diff --git a/symmetric-assemble/src/asciidoc/advanced-topics.ad b/symmetric-assemble/src/asciidoc/advanced-topics.ad index ca03fdb23c..1409ae8b40 100644 --- a/symmetric-assemble/src/asciidoc/advanced-topics.ad +++ b/symmetric-assemble/src/asciidoc/advanced-topics.ad @@ -486,6 +486,9 @@ is uploaded as an HTTP multi-part attachment. Outgoing zip files are written and Incoming zip files are staged in the filesync_incoming staging directory by source node id. The filesync_incoming/{node_id} staging directory is cleared out before each subsequent delivery of files. +A file sync pull may bundle more than one batch's zip into a single response, and an interrupted pull can resume from where it left +off instead of re-downloading everything; see <> for details. + The acknowledgement of a batch happens the same way it is acknowledged in database synchronization. The client responds with an acknowledgement as part of the response during a file push or pull. diff --git a/symmetric-assemble/src/asciidoc/advanced/transport-manager.ad b/symmetric-assemble/src/asciidoc/advanced/transport-manager.ad index 956f6cbb5a..6b4a952660 100644 --- a/symmetric-assemble/src/asciidoc/advanced/transport-manager.ad +++ b/symmetric-assemble/src/asciidoc/advanced/transport-manager.ad @@ -61,3 +61,55 @@ http.transport.manager.class=com.example.MyHttpTransportManager This applies to both the `http` and `hybrid` transport types, since the hybrid transport uses an HTTP transport manager internally for remote communication. + +==== Resumable Batch Transfer + +When a batch pull is interrupted partway through (for example, by a dropped connection through a proxy or +load balancer), SymmetricDS can resume the transfer from where it left off instead of re-downloading the +whole batch. This applies to both table (CSV) data batches and file sync (ZIP) batches, and only to the +pull direction; a push always sends the full batch. + +Resumable transfer is controlled by the `sync.http.resume.enabled` parameter, which defaults to `true`. + +[source, properties] +---- +sync.http.resume.enabled=true +---- + +Resume only takes effect for batches that are staged to disk, so it also requires `stream.to.file.enabled=true` +and a batch size over the `stream.to.file.threshold.bytes` threshold; smaller batches are streamed from memory +and always re-sent in full. + +.How it works +* Each staged batch is identified by an ETag made up of the batch's staging version, the time its staged file +was generated, and its final size. The client caches this ETag along with how much of the batch it has already +received whenever a pull attempt fails partway through. +* On the next pull attempt for that node, the client sends the cached ETag in an `If-ETag` header and how much +it already has in a `Range` header, along with the ID of the batch to resume. Table (CSV) batches use +`Range: chars=-`, a count of decoded characters, since the staged content is read and written as text; +file sync (ZIP) batches use `Range: bytes=-`, a count of raw bytes, since the staged content is copied +as-is. +* If the server still has that exact staged batch (matched by ETag) available, it responds with +`206 Partial Content`, an `ETag` header, and a `Content-Range` header, and streams only the remainder. +* If the staged batch is missing or its ETag no longer matches (for example, it was purged or re-extracted), +the server falls back to a normal `200` response with the full batch, and the client discards its partial +copy and starts over. + +Older peers (prior to version 3.18) simply do not send the resume request headers, so they always receive +full-batch behavior; the resume mechanism only activates when both sides support it. + +These `Range`/`If-ETag`/`Accept-Ranges` headers are a private convention between SymmetricDS nodes, not a +standard, cache-aware `If-Range` exchange - a node never validates a range against a shared intermediary +cache. Because of this, a proxy, load balancer, or CDN that inspects and tries to interpret or satisfy HTTP +range requests on its own (rather than passing them through unmodified) can corrupt a resumed transfer; make +sure any such intermediary in the sync path is configured to forward these headers untouched rather than act +on them. + +.File sync bundling +For file sync pulls specifically, each batch is staged as its own independent zip file, the same way table +batches already are, so that any one batch can be resumed without affecting the others. To keep bundling +multiple batches into a single pull response (subject to `transport.max.bytes.to.sync`), SymmetricDS wraps +each batch's zip in a small header — batch ID, byte length, and ETag — one after another in the response +body. A `FileSync-Format` response header tells the client that this bundling format was used; it is only +sent when the target node is version 3.18 or later, since older clients only know how to unzip a single, +unwrapped zip file per response. When the header is absent, the response is a single, unwrapped zip file. diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/common/ParameterConstants.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/common/ParameterConstants.java index 403c068b9a..df34e6839e 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/common/ParameterConstants.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/common/ParameterConstants.java @@ -266,6 +266,7 @@ private ParameterConstants() { public static final String TRANSPORT_HTTP_SESSION_EXPIRE_SECONDS = "http.session.expire.seconds"; public static final String TRANSPORT_HTTP_SESSION_MAX_COUNT = "http.session.max.count"; public static final String TRANSPORT_HTTP_USE_HEADER_SECURITY_TOKEN = "http.use.header.security.token"; + public static final String TRANSPORT_HTTP_RESUME_ENABLED = "sync.http.resume.enabled"; public static final String TRANSPORT_TYPE = "transport.type"; public static final String TRANSPORT_MAX_BYTES_TO_SYNC = "transport.max.bytes.to.sync"; public static final String TRANSPORT_MAX_FORM_KEYS = "transport.max.form.keys"; diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/extract/CountingSkippingWriter.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/extract/CountingSkippingWriter.java new file mode 100644 index 0000000000..ec1d5f254f --- /dev/null +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/extract/CountingSkippingWriter.java @@ -0,0 +1,75 @@ +/** + * Licensed to JumpMind Inc under one or more contributor + * license agreements. See the NOTICE file distributed + * with this work for additional information regarding + * copyright ownership. JumpMind Inc licenses this file + * to you under the GNU Affero General Public License, version 3.0 (AGPLv3) + * (the "License"); you may not use this file except in compliance + * with the License. + * + * You should have received a copy of the GNU Affero General Public License, + * version 3.0 (AGPLv3) along with this library; if not, see + * . + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jumpmind.symmetric.extract; + +import java.io.IOException; +import java.io.Writer; + +/** + * Wraps a destination {@link Writer}, discarding the first {@code skipCount} characters written to it and forwarding the rest, while counting the total number + * of characters seen (skipped plus forwarded). This lets a single deterministic write pass serve both a full batch resend ({@code skipCount == 0}) and a + * resumed, partial send ({@code skipCount > 0}) starting from the same point in the stream. + *

+ * The count is in decoded characters of the underlying CSV text stream, not raw network bytes: the staged resource is read and written through + * {@link java.io.Reader}/{@link java.io.Writer}, not {@link java.io.InputStream}/{@link java.io.OutputStream}, so an HTTP Range/Content-Range value used with + * this class must agree on that same unit on both the client and server side. Since both sides read the exact same staged, UTF-8 file deterministically, this + * is internally consistent even though it is not a literal byte offset per RFC 9110 Range semantics. + */ +public class CountingSkippingWriter extends Writer { + private final Writer delegate; + private final long skipCount; + private long totalCount; + + public CountingSkippingWriter(Writer delegate, long skipCount) { + this.delegate = delegate; + this.skipCount = skipCount; + } + + public long getTotalCount() { + return totalCount; + } + + @Override + public void write(char[] cbuf, int off, int len) throws IOException { + int writeOff = off; + int writeLen = len; + if (totalCount < skipCount) { + long remainingToSkip = skipCount - totalCount; + int skipInThisChunk = (int) Math.min(remainingToSkip, len); + writeOff = off + skipInThisChunk; + writeLen = len - skipInThisChunk; + } + if (writeLen > 0) { + delegate.write(cbuf, writeOff, writeLen); + } + totalCount += len; + } + + @Override + public void flush() throws IOException { + delegate.flush(); + } + + @Override + public void close() throws IOException { + delegate.close(); + } +} diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/file/FileSyncBatchEnvelope.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/file/FileSyncBatchEnvelope.java new file mode 100644 index 0000000000..93ccd00f24 --- /dev/null +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/file/FileSyncBatchEnvelope.java @@ -0,0 +1,92 @@ +/** + * Licensed to JumpMind Inc under one or more contributor + * license agreements. See the NOTICE file distributed + * with this work for additional information regarding + * copyright ownership. JumpMind Inc licenses this file + * to you under the GNU Affero General Public License, version 3.0 (AGPLv3) + * (the "License"); you may not use this file except in compliance + * with the License. + * + * You should have received a copy of the GNU Affero General Public License, + * version 3.0 (AGPLv3) along with this library; if not, see + * . + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jumpmind.symmetric.file; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; + +import org.jumpmind.symmetric.io.stage.StagedResourceETag; + +/** + * A lightweight header written immediately before each batch's complete, independently-staged zip bytes in a {@code FileSync-Format}-tagged pull response, so + * several batches can be bundled into one response while still letting the reader know exactly where one batch's zip ends and the next one's header begins — a + * purely length-based framing, no entry-by-entry inspection required. + *

+ * Wire shape is one UTF-8 text line, {@code ,,}, followed by exactly {@code zipByteLength} raw zip bytes. The ETag JSON + * itself may contain commas, so only the first two commas are treated as delimiters; everything after the second comma is taken verbatim as the ETag JSON. + */ +public class FileSyncBatchEnvelope { + private final long batchId; + private final long length; + private final StagedResourceETag etag; + + public FileSyncBatchEnvelope(long batchId, long length, StagedResourceETag etag) { + this.batchId = batchId; + this.length = length; + this.etag = etag; + } + + public long getBatchId() { + return batchId; + } + + public long getLength() { + return length; + } + + public StagedResourceETag getEtag() { + return etag; + } + + public static void writeHeader(OutputStream out, long batchId, long length, StagedResourceETag etag) throws IOException { + String line = batchId + "," + length + "," + etag.toJson() + "\n"; + out.write(line.getBytes(StandardCharsets.UTF_8)); + } + + /** + * Reads one envelope header line from {@code in}, one byte at a time so as to never consume bytes past the header's trailing newline — the caller must read + * exactly {@link #getLength()} bytes immediately afterward, so any over-read here would corrupt the following batch's zip content. + * + * @return the parsed header, or {@code null} at a clean end of stream (no more batches follow) + */ + public static FileSyncBatchEnvelope readHeader(InputStream in) throws IOException { + StringBuilder line = new StringBuilder(); + int b; + while ((b = in.read()) != -1 && b != '\n') { + line.append((char) b); + } + if (b == -1 && line.isEmpty()) { + return null; + } + String headerLine = line.toString(); + int firstComma = headerLine.indexOf(','); + int secondComma = firstComma < 0 ? -1 : headerLine.indexOf(',', firstComma + 1); + if (firstComma < 0 || secondComma < 0) { + throw new IOException("Malformed file sync envelope header: " + headerLine); + } + long batchId = Long.parseLong(headerLine.substring(0, firstComma)); + long length = Long.parseLong(headerLine.substring(firstComma + 1, secondComma)); + StagedResourceETag etag = StagedResourceETag.fromJson(headerLine.substring(secondComma + 1)); + return new FileSyncBatchEnvelope(batchId, length, etag); + } +} diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/file/FileSyncPullResult.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/file/FileSyncPullResult.java new file mode 100644 index 0000000000..33a3a267e3 --- /dev/null +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/file/FileSyncPullResult.java @@ -0,0 +1,150 @@ +/** + * Licensed to JumpMind Inc under one or more contributor + * license agreements. See the NOTICE file distributed + * with this work for additional information regarding + * copyright ownership. JumpMind Inc licenses this file + * to you under the GNU Affero General Public License, version 3.0 (AGPLv3) + * (the "License"); you may not use this file except in compliance + * with the License. + * + * You should have received a copy of the GNU Affero General Public License, + * version 3.0 (AGPLv3) along with this library; if not, see + * . + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jumpmind.symmetric.file; + +import java.util.List; + +import org.jumpmind.symmetric.io.stage.IStagedResource; +import org.jumpmind.symmetric.io.stage.StagedResourceETag; +import org.jumpmind.symmetric.model.OutgoingBatch; + +/** + * The outcome of {@link org.jumpmind.symmetric.service.IFileSyncService#prepareFilesForPull}, carrying both what the servlet handler needs to set response + * headers/status, and what {@link org.jumpmind.symmetric.service.IFileSyncService#writeFilesForPull} needs to stream the previously-staged bytes afterward. + * Split from a single combined call so the handler can set headers on the servlet response before any bytes are written to it - setting a header on an + * already-committed response is a silent no-op, which previously meant the {@code FileSync-Format} header was never actually sent to the client. + *

+ * {@code resumeEtag} is non-null only when this response served (or attempted to serve) exactly one specific, previously-interrupted batch by request; it is + * {@code null} for a normal, non-resume pull. {@code allRequestedBatches} is only meaningful for a normal (non-resume) pull - it is the full candidate list + * {@code batches} was selected from, needed by {@link org.jumpmind.symmetric.service.IFileSyncService#writeFilesForPull} to mark them loaded. + */ +public class FileSyncPullResult { + private final List batches; + private final List allRequestedBatches; + private final List stagedResources; + private final boolean isEnvelopeFormatUsed; + private final boolean isPartialContent; + private final StagedResourceETag resumeEtag; + private final long totalSize; + private final long skipCount; + + private FileSyncPullResult(Builder builder) { + this.batches = builder.batches; + this.allRequestedBatches = builder.allRequestedBatches; + this.stagedResources = builder.stagedResources; + this.isEnvelopeFormatUsed = builder.isEnvelopeFormatUsed; + this.isPartialContent = builder.isPartialContent; + this.resumeEtag = builder.resumeEtag; + this.totalSize = builder.totalSize; + this.skipCount = builder.skipCount; + } + + public static Builder builder() { + return new Builder(); + } + + public List getBatches() { + return batches; + } + + public List getAllRequestedBatches() { + return allRequestedBatches; + } + + public List getStagedResources() { + return stagedResources; + } + + public boolean isEnvelopeFormatUsed() { + return isEnvelopeFormatUsed; + } + + public boolean isPartialContent() { + return isPartialContent; + } + + public StagedResourceETag getResumeEtag() { + return resumeEtag; + } + + public long getTotalSize() { + return totalSize; + } + + public long getSkipCount() { + return skipCount; + } + + public static class Builder { + private List batches; + private List allRequestedBatches; + private List stagedResources; + private boolean isEnvelopeFormatUsed; + private boolean isPartialContent; + private StagedResourceETag resumeEtag; + private long totalSize; + private long skipCount; + + public Builder batches(List batches) { + this.batches = batches; + return this; + } + + public Builder allRequestedBatches(List allRequestedBatches) { + this.allRequestedBatches = allRequestedBatches; + return this; + } + + public Builder stagedResources(List stagedResources) { + this.stagedResources = stagedResources; + return this; + } + + public Builder envelopeFormatUsed(boolean isEnvelopeFormatUsed) { + this.isEnvelopeFormatUsed = isEnvelopeFormatUsed; + return this; + } + + public Builder partialContent(boolean isPartialContent) { + this.isPartialContent = isPartialContent; + return this; + } + + public Builder resumeEtag(StagedResourceETag resumeEtag) { + this.resumeEtag = resumeEtag; + return this; + } + + public Builder totalSize(long totalSize) { + this.totalSize = totalSize; + return this; + } + + public Builder skipCount(long skipCount) { + this.skipCount = skipCount; + return this; + } + + public FileSyncPullResult build() { + return new FileSyncPullResult(this); + } + } +} diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/io/stage/SimpleStagingDataWriter.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/io/stage/SimpleStagingDataWriter.java index 2c11ea04a3..e7a8564a14 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/io/stage/SimpleStagingDataWriter.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/io/stage/SimpleStagingDataWriter.java @@ -33,6 +33,7 @@ import org.jumpmind.symmetric.AbstractSymmetricEngine; import org.jumpmind.symmetric.ISymmetricEngine; import org.jumpmind.symmetric.common.Constants; +import org.jumpmind.symmetric.common.ParameterConstants; import org.jumpmind.symmetric.csv.CsvReader; import org.jumpmind.symmetric.io.data.Batch; import org.jumpmind.symmetric.io.data.Batch.BatchType; @@ -43,6 +44,8 @@ import org.jumpmind.symmetric.io.stage.IStagedResource.State; import org.jumpmind.symmetric.model.ProcessInfo; import org.jumpmind.symmetric.model.ProcessInfo.ProcessStatus; +import org.jumpmind.symmetric.transport.http.IHttpResumeCache; +import org.jumpmind.symmetric.transport.http.ResumeCacheEntry; import org.jumpmind.util.Statistics; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,22 +68,25 @@ public class SimpleStagingDataWriter { protected Batch batch; protected long invalidLineCount; protected Exception exception; + protected ResumeCacheEntry resumeEntry; + protected StagedResourceETag currentBatchEtag; + protected long stagedCharCount; - public SimpleStagingDataWriter(ProcessInfo processInfo, BufferedReader reader, ISymmetricEngine engine, String category, long memoryThresholdInBytes, - BatchType batchType, String sourceNodeId, String targetNodeId, DataContext context, IProtocolDataWriterListener... listeners) { - this.reader = new CsvReader(reader); + private SimpleStagingDataWriter(Builder builder) { + this.reader = new CsvReader(builder.reader); this.reader.setEscapeMode(CsvReader.ESCAPE_MODE_BACKSLASH); this.reader.setSafetySwitch(false); - this.engine = engine; - this.stagingManager = engine.getStagingManager(); - this.memoryThresholdInBytes = memoryThresholdInBytes; - this.category = category; - this.batchType = batchType; - this.sourceNodeId = sourceNodeId; - this.targetNodeId = targetNodeId; - this.listeners = listeners; - this.context = context; - this.processInfo = processInfo; + this.engine = builder.engine; + this.stagingManager = builder.engine.getStagingManager(); + this.memoryThresholdInBytes = builder.memoryThresholdInBytes; + this.category = builder.category; + this.batchType = builder.batchType; + this.sourceNodeId = builder.sourceNodeId; + this.targetNodeId = builder.targetNodeId; + this.listeners = builder.listeners; + this.context = builder.context; + this.processInfo = builder.processInfo; + this.resumeEntry = builder.resumeEntry; } public void process() throws IOException { @@ -95,6 +101,9 @@ public void process() throws IOException { String batchStatsColumnsLine = null; String batchStatsLine = null; Statistics batchStats = null; + if (resumeEntry != null) { + resource = beginResumedBatch(); + } while (reader.readRecord()) { line = reader.getRawRecord(); if (line.startsWith(CsvConstants.CATALOG)) { @@ -145,6 +154,8 @@ public void process() throws IOException { } resource = stagingManager.create(category, location, batch.getBatchId()); writer = resource.getWriter(memoryThresholdInBytes); + currentBatchEtag = null; + stagedCharCount = 0; writeLine(nodeLine); writeLine(binaryLine); writeLine(channelLine); @@ -154,6 +165,8 @@ public void process() throws IOException { listener.start(context, batch); } } + } else if (line.startsWith(CsvConstants.ETAG)) { + currentBatchEtag = StagedResourceETag.fromJson(getArgLine(line)); } else if (line.startsWith(CsvConstants.COMMIT)) { if (writer != null) { writeLine(line); @@ -169,9 +182,11 @@ public void process() throws IOException { listener.end(context, batch, resource); } } + clearResumeCacheEntry(batch.getBatchId()); } batchStats = null; resource = null; + currentBatchEtag = null; } else if (line.startsWith(CsvConstants.RETRY)) { batch = new Batch(batchType, Long.parseLong(getArgLine(line)), getArgLine(channelLine), getBinaryEncoding(binaryLine), getArgLine(nodeLine), targetNodeId, false); @@ -192,6 +207,8 @@ public void process() throws IOException { resource = null; writer = null; } + currentBatchEtag = null; + stagedCharCount = 0; if (log.isDebugEnabled()) { debugLine(nodeLine); debugLine(binaryLine); @@ -251,6 +268,7 @@ public void process() throws IOException { writer.append(line, i, end < size ? end : size); } writer.append("\n"); + stagedCharCount += size + 1; } else { writeLine(line); } @@ -275,7 +293,11 @@ public void process() throws IOException { exception = ex; } if (resource != null) { - resource.delete(); + if (isResumableInterruption(ex, resource)) { + registerForResume(resource); + } else { + resource.delete(); + } } processInfo.setStatus(ProcessStatus.ERROR); /* @@ -321,6 +343,7 @@ protected void writeLine(String line) throws IOException { if (writer != null) { writer.write(line); writer.write("\n"); + stagedCharCount += line.length() + 1; } else { exception = new ProtocolException("Batch data is corrupt from node " + sourceNodeId + " because no batch ID was present"); processInfo.setStatus(ProcessStatus.ERROR); @@ -367,6 +390,90 @@ protected IStagedResource getStagedResource() { return resource; } + /** + * A confirmed resumed ({@code 206}) response contains only the remaining row data for the one batch being resumed, not its preamble + * (NODEID/BINARY/CHANNEL/BATCH/ETAG lines) — that part was already received and staged on the prior, interrupted attempt. So unlike a fresh batch, there's + * no {@code BATCH} line here to trigger the normal setup; instead, reconstruct the batch's identity from {@link #resumeEntry} and reopen its existing local + * partial resource in append mode, before any lines are read from the wire. + * + * @return the reopened local resource, or {@code null} if it was missing or already finalized, in which case this pull cannot complete that batch and a + * later pull will retry it in full + */ + protected IStagedResource beginResumedBatch() { + BinaryEncoding binaryEncoding = resumeEntry.getBinaryEncoding() != null ? BinaryEncoding.valueOf(resumeEntry.getBinaryEncoding()) : null; + Batch candidateBatch = new Batch(batchType, resumeEntry.getBatchId(), resumeEntry.getChannelId(), + binaryEncoding, sourceNodeId, targetNodeId, false); + IStagedResource existingResource = stagingManager.find(category, candidateBatch.getStagedLocation(), candidateBatch.getBatchId()); + if (existingResource == null || existingResource.getState() != State.CREATE) { + log.warn("Resume requested for batch {} from node {}, but the local partial staged resource was missing or already finalized ({}). " + + "This pull cannot complete that batch; a subsequent pull will retry it in full.", + candidateBatch.getBatchId(), sourceNodeId, existingResource == null ? "not found" : existingResource.getState()); + clearResumeCacheEntry(resumeEntry.getBatchId()); + return null; + } + batch = candidateBatch; + writer = existingResource.getWriter(memoryThresholdInBytes, true); + currentBatchEtag = resumeEntry.getEtag(); + stagedCharCount = 0; + processInfo.setCurrentBatchId(batch.getBatchId()); + processInfo.setCurrentBatchStartTime(new Date()); + processInfo.incrementBatchCount(); + processInfo.setCurrentDataCount(0); + processInfo.setTotalDataCount(0); + if (listeners != null) { + for (IProtocolDataWriterListener listener : listeners) { + listener.start(context, batch); + } + } + return existingResource; + } + + /** + * @return whether {@code ex} represents a connection-level failure (not a data/protocol error) for a batch that's genuinely eligible to be preserved for a + * resumed retry: resume is enabled, the batch's staged content is file-backed, and its ETag was captured before the interruption + */ + protected boolean isResumableInterruption(Exception ex, IStagedResource resource) { + return ex instanceof IOException && batch != null && currentBatchEtag != null && resource.isFileResource() + && engine.getParameterService().is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED) + && getResumeCache() != null; + } + + protected void registerForResume(IStagedResource resource) { + resource.close(); + long receivedCount = (resumeEntry != null ? resumeEntry.getReceivedCount() : 0) + stagedCharCount; + IHttpResumeCache resumeCache = getResumeCache(); + if (resumeCache == null) { + return; + } + resumeCache.put(sourceNodeId, batch.getBatchId(), ResumeCacheEntry.builder() + .nodeId(sourceNodeId) + .batchId(batch.getBatchId()) + .etag(currentBatchEtag) + .receivedCount(receivedCount) + .channelId(batch.getChannelId()) + .binaryEncoding(batch.getBinaryEncoding() != null ? batch.getBinaryEncoding().name() : null) + .cachedAtTime(System.currentTimeMillis()) + .queue(processInfo.getQueue()) + .build()); + log.info("Preserving partially-received batch {} from node {} for a resumed retry ({} characters received).", + batch.getBatchId(), sourceNodeId, receivedCount); + } + + protected void clearResumeCacheEntry(long batchId) { + IHttpResumeCache resumeCache = getResumeCache(); + if (resumeCache != null) { + resumeCache.remove(sourceNodeId, batchId); + } + } + + protected IHttpResumeCache getResumeCache() { + return engine.getTransportManager() != null ? engine.getTransportManager().getResumeCache() : null; + } + + public static Builder builder() { + return new Builder(); + } + static class TableLine { String catalogLine; String schemaLine; @@ -395,4 +502,82 @@ public int hashCode() { return (catalogLine + "." + schemaLine + "." + tableLine).hashCode(); } } + + public static class Builder { + private ProcessInfo processInfo; + private BufferedReader reader; + private ISymmetricEngine engine; + private String category; + private long memoryThresholdInBytes; + private BatchType batchType; + private String sourceNodeId; + private String targetNodeId; + private DataContext context; + private ResumeCacheEntry resumeEntry; + private IProtocolDataWriterListener[] listeners = new IProtocolDataWriterListener[0]; + + public Builder processInfo(ProcessInfo processInfo) { + this.processInfo = processInfo; + return this; + } + + public Builder reader(BufferedReader reader) { + this.reader = reader; + return this; + } + + public Builder engine(ISymmetricEngine engine) { + this.engine = engine; + return this; + } + + public Builder category(String category) { + this.category = category; + return this; + } + + public Builder memoryThresholdInBytes(long memoryThresholdInBytes) { + this.memoryThresholdInBytes = memoryThresholdInBytes; + return this; + } + + public Builder batchType(BatchType batchType) { + this.batchType = batchType; + return this; + } + + public Builder sourceNodeId(String sourceNodeId) { + this.sourceNodeId = sourceNodeId; + return this; + } + + public Builder targetNodeId(String targetNodeId) { + this.targetNodeId = targetNodeId; + return this; + } + + public Builder context(DataContext context) { + this.context = context; + return this; + } + + /** + * @param resumeEntry + * non-null only when this pull is a confirmed resume of one specific, previously-interrupted batch; {@code null} for every normal (fresh) + * pull + */ + public Builder resumeEntry(ResumeCacheEntry resumeEntry) { + this.resumeEntry = resumeEntry; + return this; + } + + public Builder listeners(IProtocolDataWriterListener... listeners) { + this.listeners = listeners; + return this; + } + + public SimpleStagingDataWriter build() { + return new SimpleStagingDataWriter(this); + } + } } diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/IDataExtractorService.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/IDataExtractorService.java index e105efab76..71f9532e1a 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/IDataExtractorService.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/IDataExtractorService.java @@ -26,6 +26,7 @@ import org.jumpmind.db.sql.ISqlTransaction; import org.jumpmind.symmetric.io.data.writer.StructureDataWriter.PayloadType; +import org.jumpmind.symmetric.io.stage.IStagedResource; import org.jumpmind.symmetric.io.stage.StagingFileLock; import org.jumpmind.symmetric.model.ExtractRequest; import org.jumpmind.symmetric.model.Node; @@ -59,6 +60,20 @@ public boolean extractBatchRange(Writer writer, String nodeId, Date startBatchTi public boolean extractOnlyOutgoingBatch(String nodeId, long batchId, Writer writer); + /** + * @return the batch's staged resource if it exists and is fully staged, usable for a resumed pull; otherwise {@code null} + */ + public IStagedResource getStagedResourceForResume(OutgoingBatch batch); + + /** + * Streams a single batch's staged content, optionally skipping the first {@code skipCount} already-received characters, for a resumed pull. See + * {@code DataExtractorService.extractSingleBatchForResume} for details. + * + * @return the total number of characters in the batch (skipped plus forwarded) + */ + public long extractSingleBatchForResume(OutgoingBatch batch, IStagedResource stagedResource, Writer destination, + long skipCount, ProcessInfo processInfo); + public RemoteNodeStatuses queueWork(boolean force); public ExtractRequest requestExtractRequest(ISqlTransaction transaction, String nodeId, String channelId, TriggerRouter triggerRouter, long startBatchId, diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/IFileSyncService.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/IFileSyncService.java index 86159ce75c..7a80b4e3b4 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/IFileSyncService.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/IFileSyncService.java @@ -27,6 +27,7 @@ import org.jumpmind.db.sql.ISqlTransaction; import org.jumpmind.symmetric.file.DirectorySnapshot; +import org.jumpmind.symmetric.file.FileSyncPullResult; import org.jumpmind.symmetric.model.FileSnapshot; import org.jumpmind.symmetric.model.FileTrigger; import org.jumpmind.symmetric.model.FileTriggerRouter; @@ -89,6 +90,28 @@ public interface IFileSyncService { public List sendFiles(ProcessInfo processInfo, Node node, IOutgoingTransport outgoingTransport); + /** + * Same overall purpose as {@link #sendFiles(ProcessInfo, Node, IOutgoingTransport)}, but used only by the pull path and split into two phases so the + * servlet handler can set response headers/status (ETag, Content-Range, {@code FileSync-Format}) based on the returned {@link FileSyncPullResult} + * before any bytes are written to the response - a response is committed the moment its output stream is written to, after which setting a header + * on it is a silent no-op. + *

+ * This phase decides whether to resume one specific, previously-interrupted batch (when {@code batchIdParam} is non-blank), extracts/stages whichever + * batches will be sent, and computes the envelope/partial-content decisions - it performs no network writes. Pass the result to + * {@link #writeFilesForPull(ProcessInfo, FileSyncPullResult, IOutgoingTransport)} to actually stream the bytes. + * {@link #sendFiles(ProcessInfo, Node, IOutgoingTransport)} remains unchanged for the push path, which has no equivalent resume mechanism. + */ + public FileSyncPullResult prepareFilesForPull(ProcessInfo processInfo, Node targetNode, String batchIdParam, + String ifETagHeader, String rangeHeader); + + /** + * Streams the bytes described by a {@link FileSyncPullResult} previously returned from + * {@link #prepareFilesForPull(ProcessInfo, Node, String, String, String)} to the given transport, and performs the associated batch bookkeeping (marking + * batches loaded, cleaning up staged resources) once the write completes. Must be called only after the caller has finished setting response + * headers/status. + */ + public void writeFilesForPull(ProcessInfo processInfo, FileSyncPullResult result, IOutgoingTransport outgoingTransport); + public void acknowledgeFiles(OutgoingBatch outgoingBatch); public boolean refreshFromDatabase(); diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataExtractorService.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataExtractorService.java index 42169a55da..7ab138b276 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataExtractorService.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataExtractorService.java @@ -88,6 +88,7 @@ import org.jumpmind.symmetric.common.ParameterConstants; import org.jumpmind.symmetric.common.TableConstants; import org.jumpmind.symmetric.ext.IReloadQueueThreadAssigner; +import org.jumpmind.symmetric.extract.CountingSkippingWriter; import org.jumpmind.symmetric.extract.ExtractDataReaderFactory; import org.jumpmind.symmetric.extract.IExtractDataReaderFactory; import org.jumpmind.symmetric.extract.MultiBatchStagingWriter; @@ -119,6 +120,7 @@ import org.jumpmind.symmetric.io.stage.IStagedResource; import org.jumpmind.symmetric.io.stage.IStagedResource.State; import org.jumpmind.symmetric.io.stage.IStagingManager; +import org.jumpmind.symmetric.io.stage.StagedResourceETag; import org.jumpmind.symmetric.io.stage.StagingFileLock; import org.jumpmind.symmetric.io.stage.StagingLowFreeSpace; import org.jumpmind.symmetric.model.AbstractBatch.Status; @@ -1296,6 +1298,42 @@ protected boolean isPreviouslyExtracted(OutgoingBatch currentBatch, boolean acqu return false; } + @Override + public IStagedResource getStagedResourceForResume(OutgoingBatch batch) { + if (batch == null) { + return null; + } + return getStagedResource(batch); + } + + /** + * Streams a single batch's staged content to {@code destination}, optionally skipping the first {@code skipCount} characters already known to have been + * received by the client on a prior, interrupted attempt. Reuses {@link #transferFromStaging} unchanged so a full resend ({@code skipCount == 0}) and a + * resumed partial send share the exact same deterministic framing/stats-injection logic. + *

+ * {@code skipCount} (and the returned total) are counts of decoded characters of the staged UTF-8 CSV text, not raw HTTP bytes, since the staging layer is + * read and written through a Reader/Writer pair, not an InputStream/OutputStream. Both sides of a resume exchange must agree on this same unit. + * + * @return the total number of characters in the batch (skipped plus forwarded), for use in a {@code Content-Range} response header + */ + @Override + public long extractSingleBatchForResume(OutgoingBatch batch, IStagedResource stagedResource, Writer destination, + long skipCount, ProcessInfo processInfo) { + batch.setSentCount(batch.getSentCount() + 1); + outgoingBatchService.updateOutgoingBatch(batch); + CountingSkippingWriter countingWriter = new CountingSkippingWriter(destination, skipCount); + BufferedWriter bufferedWriter = new BufferedWriter(countingWriter); + Channel channel = configurationService.getChannel(batch.getChannelId()); + transferFromStaging(new StagedBatchTransferRequest(ExtractMode.FOR_SYM_CLIENT, BatchType.EXTRACT, batch, false, stagedResource, skipCount > 0), + bufferedWriter, channel.getMaxKBytesPerSecond(), processInfo); + try { + bufferedWriter.flush(); + } catch (IOException e) { + throw new IoException(e); + } + return countingWriter.getTotalCount(); + } + protected boolean isRetry(OutgoingBatch currentBatch, Node remoteNode) { if (currentBatch.getSentCount() > 0 && currentBatch.getStatus() != OutgoingBatch.Status.RS && currentBatch.getStatus() != OutgoingBatch.Status.IG) { boolean offline = parameterService.is(ParameterConstants.NODE_OFFLINE, false); @@ -1374,9 +1412,8 @@ protected OutgoingBatch sendOutgoingBatch(ProcessInfo processInfo, Node targetNo } } Channel channel = configurationService.getChannel(currentBatch.getChannelId()); - DataContext ctx = new DataContext(); - transferFromStaging(mode, BatchType.EXTRACT, currentBatch, isRetry, extractedBatch, writer, ctx, - channel.getMaxKBytesPerSecond(), processInfo); + transferFromStaging(new StagedBatchTransferRequest(mode, BatchType.EXTRACT, currentBatch, isRetry, extractedBatch, false), + writer, channel.getMaxKBytesPerSecond(), processInfo); } else { IDataReader dataReader = new ProtocolDataReader(BatchType.EXTRACT, currentBatch.getNodeId(), extractedBatch); @@ -1408,100 +1445,17 @@ protected OutgoingBatch sendOutgoingBatch(ProcessInfo processInfo, Node targetNo return currentBatch; } - protected void transferFromStaging(ExtractMode mode, BatchType batchType, OutgoingBatch batch, boolean isRetry, IStagedResource stagedResource, - BufferedWriter writer, DataContext context, BigDecimal maxKBytesPerSec, ProcessInfo processInfo) { - final int MAX_WRITE_LENGTH = 32768; + protected void transferFromStaging(StagedBatchTransferRequest request, BufferedWriter writer, BigDecimal maxKBytesPerSec, ProcessInfo processInfo) { + OutgoingBatch batch = request.getBatch(); + IStagedResource stagedResource = request.getStagedResource(); BufferedReader reader = stagedResource.getReader(); try { // Retry means we've sent this batch before, so let's ask to // retry the batch from the target's staging - if (isRetry) { - String line = null; - while ((line = reader.readLine()) != null) { - if (line.startsWith(CsvConstants.BATCH)) { - if (nodeService.findNode(batch.getNodeId(), true).isVersionGreaterThanOrEqualTo(3, 9, 0)) { - writer.write(getBatchStatsColumns()); - writer.newLine(); - writer.write(getBatchStats(batch)); - writer.newLine(); - } - writer.write(CsvConstants.RETRY + "," + batch.getBatchId()); - writer.newLine(); - writer.write(CsvConstants.COMMIT + "," + batch.getBatchId()); - writer.newLine(); - break; - } else { - writer.write(line); - writer.newLine(); - } - } - writer.flush(); - processInfo.setCurrentDataCount(batch.getDataRowCount()); + if (request.isRetry()) { + sendRetryNotification(reader, writer, batch, processInfo); } else { - long totalBytes = stagedResource.getSize(); - long totalCharsRead = 0, totalBytesRead = 0; - int numCharsRead = 0, numBytesRead = 0; - long startTime = System.currentTimeMillis(), ts = startTime, bts = startTime; - boolean isThrottled = maxKBytesPerSec != null && maxKBytesPerSec.compareTo(BigDecimal.ZERO) > 0; - long totalThrottleTime = 0; - int bufferSize = MAX_WRITE_LENGTH; - if (isThrottled) { - bufferSize = maxKBytesPerSec.multiply(new BigDecimal(1024)).intValue(); - } - char[] buffer = new char[bufferSize]; - boolean batchStatsWritten = false; - String prevBuffer = ""; - long batchStatusUpdateMillis = parameterService.getLong(ParameterConstants.OUTGOING_BATCH_UPDATE_STATUS_MILLIS); - boolean is39orNewer = nodeService.findNode(batch.getNodeId(), true).isVersionGreaterThanOrEqualTo(3, 9, 0); - while ((numCharsRead = reader.read(buffer)) != -1) { - if (!batchStatsWritten && is39orNewer) { - batchStatsWritten = writeBatchStats(writer, buffer, numCharsRead, prevBuffer, batch); - prevBuffer = new String(buffer); - } else { - writer.write(buffer, 0, numCharsRead); - } - totalCharsRead += numCharsRead; - if (Thread.currentThread().isInterrupted()) { - throw new IoException("This thread was interrupted"); - } - if (System.currentTimeMillis() - ts > batchStatusUpdateMillis && batch.getStatus() != Status.SE && batch.getStatus() != Status.RS) { - changeBatchStatus(Status.SE, batch, mode); - } - if (System.currentTimeMillis() - ts > LOG_PROCESS_SUMMARY_THRESHOLD) { - log.info( - "Batch '{}', for node '{}', for process 'send from stage' has been processing for {} seconds. " - + "The following stats have been gathered: {}", - new Object[] { batch.getBatchId(), batch.getNodeId(), (System.currentTimeMillis() - startTime) / 1000, - "CHARS=" + totalCharsRead }); - ts = System.currentTimeMillis(); - } - if (isThrottled) { - numBytesRead += new String(buffer, 0, numCharsRead).getBytes().length; - totalBytesRead += numBytesRead; - if (numBytesRead >= bufferSize) { - long expectedMillis = (long) (((numBytesRead / 1024f) / maxKBytesPerSec.floatValue()) * 1000); - long actualMillis = System.currentTimeMillis() - bts; - if (actualMillis < expectedMillis) { - totalThrottleTime += expectedMillis - actualMillis; - Thread.sleep(expectedMillis - actualMillis); - } - numBytesRead = 0; - bts = System.currentTimeMillis(); - } - } else { - totalBytesRead += new String(buffer, 0, numCharsRead).getBytes().length; - } - processInfo.setCurrentDataCount((long) ((totalBytesRead / (double) totalBytes) * batch.getDataRowCount())); - } - if (batch.getSentCount() == 1) { - statisticManager.incrementDataSent(batch.getChannelId(), batch.getDataRowCount()); - statisticManager.incrementDataBytesSent(batch.getChannelId(), totalBytesRead); - } - if (log.isDebugEnabled() && totalThrottleTime > 0) { - log.debug("Batch '{}' for node '{}' took {}ms for {} bytes and was throttled for {}ms because limit is set to {} KB/s", - batch.getBatchId(), batch.getNodeId(), (System.currentTimeMillis() - startTime), totalBytesRead, - totalThrottleTime, maxKBytesPerSec); - } + sendStagedContent(request, reader, writer, maxKBytesPerSec, processInfo); } if (writer instanceof BatchBufferedWriter) { ((BatchBufferedWriter) writer).getBatchIds().add(batch.getBatchId()); @@ -1514,6 +1468,129 @@ protected void transferFromStaging(ExtractMode mode, BatchType batchType, Outgoi } } + private void sendRetryNotification(BufferedReader reader, BufferedWriter writer, OutgoingBatch batch, ProcessInfo processInfo) throws IOException { + String line = null; + while ((line = reader.readLine()) != null) { + if (line.startsWith(CsvConstants.BATCH)) { + writeRetryBatchLine(writer, batch); + break; + } else { + writer.write(line); + writer.newLine(); + } + } + writer.flush(); + processInfo.setCurrentDataCount(batch.getDataRowCount()); + } + + private void writeRetryBatchLine(BufferedWriter writer, OutgoingBatch batch) throws IOException { + if (nodeService.findNode(batch.getNodeId(), true).isVersionGreaterThanOrEqualTo(3, 9, 0)) { + writer.write(getBatchStatsColumns()); + writer.newLine(); + writer.write(getBatchStats(batch)); + writer.newLine(); + } + writer.write(CsvConstants.RETRY + "," + batch.getBatchId()); + writer.newLine(); + writer.write(CsvConstants.COMMIT + "," + batch.getBatchId()); + writer.newLine(); + } + + private void sendStagedContent(StagedBatchTransferRequest request, BufferedReader reader, BufferedWriter writer, BigDecimal maxKBytesPerSec, + ProcessInfo processInfo) throws IOException, InterruptedException { + final int maxWriteLength = 32768; + OutgoingBatch batch = request.getBatch(); + IStagedResource stagedResource = request.getStagedResource(); + boolean isThrottled = maxKBytesPerSec != null && maxKBytesPerSec.compareTo(BigDecimal.ZERO) > 0; + int bufferSize = isThrottled ? maxKBytesPerSec.multiply(new BigDecimal(1024)).intValue() : maxWriteLength; + char[] buffer = new char[bufferSize]; + boolean is39orNewer = nodeService.findNode(batch.getNodeId(), true).isVersionGreaterThanOrEqualTo(3, 9, 0); + StagedResourceETag resumeEtag = request.isSuppressPreambleExtras() ? null : getResumeEtagIfEligible(batch, stagedResource); + TransferProgress progress = new TransferProgress(request, is39orNewer, resumeEtag, isThrottled, bufferSize, maxKBytesPerSec, + stagedResource.getSize()); + int numCharsRead; + while ((numCharsRead = reader.read(buffer)) != -1) { + writeChunk(writer, buffer, numCharsRead, progress); + progress.totalCharsRead += numCharsRead; + checkNotInterrupted(); + updateBatchStatusIfDue(progress); + logProgressIfDue(progress); + applyThrottle(buffer, numCharsRead, progress); + processInfo.setCurrentDataCount((long) ((progress.totalBytesRead / (double) progress.totalBytes) * batch.getDataRowCount())); + } + recordSentStatistics(progress); + logThrottleSummary(progress); + } + + private void writeChunk(BufferedWriter writer, char[] buffer, int numCharsRead, TransferProgress progress) throws IOException { + if (!progress.isSuppressPreambleExtras && !progress.batchPreambleExtrasWritten && (progress.is39orNewer || progress.resumeEtag != null)) { + progress.batchPreambleExtrasWritten = writeBatchPreambleExtras(writer, buffer, numCharsRead, progress.prevBuffer, progress.batch, + progress.is39orNewer, progress.resumeEtag); + progress.prevBuffer = new String(buffer); + } else { + writer.write(buffer, 0, numCharsRead); + } + } + + private void checkNotInterrupted() { + if (Thread.currentThread().isInterrupted()) { + throw new IoException("This thread was interrupted"); + } + } + + private void updateBatchStatusIfDue(TransferProgress progress) { + long batchStatusUpdateMillis = parameterService.getLong(ParameterConstants.OUTGOING_BATCH_UPDATE_STATUS_MILLIS); + if (System.currentTimeMillis() - progress.ts > batchStatusUpdateMillis && progress.batch.getStatus() != Status.SE + && progress.batch.getStatus() != Status.RS) { + changeBatchStatus(Status.SE, progress.batch, progress.mode); + } + } + + private void logProgressIfDue(TransferProgress progress) { + if (System.currentTimeMillis() - progress.ts > LOG_PROCESS_SUMMARY_THRESHOLD) { + log.info( + "Batch '{}', for node '{}', for process 'send from stage' has been processing for {} seconds. " + + "The following stats have been gathered: CHARS={}", + progress.batch.getBatchId(), progress.batch.getNodeId(), (System.currentTimeMillis() - progress.startTime) / 1000, + progress.totalCharsRead); + progress.ts = System.currentTimeMillis(); + } + } + + private void applyThrottle(char[] buffer, int numCharsRead, TransferProgress progress) throws InterruptedException { + if (progress.isThrottled) { + progress.numBytesRead += new String(buffer, 0, numCharsRead).getBytes().length; + progress.totalBytesRead += progress.numBytesRead; + if (progress.numBytesRead >= progress.bufferSize) { + long expectedMillis = (long) (((progress.numBytesRead / 1024f) / progress.maxKBytesPerSec.floatValue()) * 1000); + long actualMillis = System.currentTimeMillis() - progress.bts; + if (actualMillis < expectedMillis) { + progress.totalThrottleTime += expectedMillis - actualMillis; + Thread.sleep(expectedMillis - actualMillis); + } + progress.numBytesRead = 0; + progress.bts = System.currentTimeMillis(); + } + } else { + progress.totalBytesRead += new String(buffer, 0, numCharsRead).getBytes().length; + } + } + + private void recordSentStatistics(TransferProgress progress) { + if (progress.batch.getSentCount() == 1) { + statisticManager.incrementDataSent(progress.batch.getChannelId(), progress.batch.getDataRowCount()); + statisticManager.incrementDataBytesSent(progress.batch.getChannelId(), progress.totalBytesRead); + } + } + + private void logThrottleSummary(TransferProgress progress) { + if (log.isDebugEnabled() && progress.totalThrottleTime > 0) { + log.debug("Batch '{}' for node '{}' took {}ms for {} bytes and was throttled for {}ms because limit is set to {} KB/s", + progress.batch.getBatchId(), progress.batch.getNodeId(), (System.currentTimeMillis() - progress.startTime), progress.totalBytesRead, + progress.totalThrottleTime, progress.maxKBytesPerSec); + } + } + protected int findStatsIndex(String bufferString, String prevBuffer) { int index = -1; String fullBuffer = prevBuffer + bufferString; @@ -1621,6 +1698,16 @@ public int cancelExtractRequests(long loadId) { protected boolean writeBatchStats(BufferedWriter writer, char[] buffer, int bufferSize, String prevBuffer, OutgoingBatch batch) throws IOException { + return writeBatchPreambleExtras(writer, buffer, bufferSize, prevBuffer, batch, true, null); + } + + /** + * Injects any extra preamble lines (batch stats and/or a resume {@code ETAG}) immediately after the {@code BATCH,} line, in the same single pass over + * {@code buffer} so each character is written exactly once. Passing {@code includeStats=false} and a {@code null} etag reproduces the original, pre-resume + * behavior byte-for-byte other than skipping the injection entirely. + */ + protected boolean writeBatchPreambleExtras(BufferedWriter writer, char[] buffer, int bufferSize, String prevBuffer, OutgoingBatch batch, + boolean includeStats, StagedResourceETag etag) throws IOException { String bufferString = new String(buffer); int index = findStatsIndex(bufferString, prevBuffer); if (index > 0) { @@ -1628,9 +1715,16 @@ protected boolean writeBatchStats(BufferedWriter writer, char[] buffer, int buff writer.write(prefix, 0, index); } if (index > -1) { - String stats = getBatchStatsColumns() + System.lineSeparator() + getBatchStats(batch) + System.lineSeparator(); - char statsBuffer[] = stats.toCharArray(); - writer.write(statsBuffer, 0, statsBuffer.length); + StringBuilder extras = new StringBuilder(); + if (includeStats) { + extras.append(getBatchStatsColumns()).append(System.lineSeparator()) + .append(getBatchStats(batch)).append(System.lineSeparator()); + } + if (etag != null) { + extras.append(CsvConstants.ETAG).append(",").append(etag.toJson()).append(System.lineSeparator()); + } + char[] extrasBuffer = extras.toString().toCharArray(); + writer.write(extrasBuffer, 0, extrasBuffer.length); char suffix[] = Arrays.copyOfRange(buffer, index, buffer.length); writer.write(suffix, 0, bufferSize - index); } else { @@ -1639,6 +1733,22 @@ protected boolean writeBatchStats(BufferedWriter writer, char[] buffer, int buff return index > -1; } + /** + * A batch is eligible for a proactive resume {@code ETAG} only when resume is enabled, the target node understands it (3.18+), and the batch's staged + * content is file-backed and fully staged ({@link State#DONE}) — the same eligibility a resumed pull itself requires in {@code PullUriHandler}. + */ + protected StagedResourceETag getResumeEtagIfEligible(OutgoingBatch batch, IStagedResource stagedResource) { + if (!parameterService.is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED) + || stagedResource == null || stagedResource.getState() != State.DONE || !stagedResource.isFileResource()) { + return null; + } + Node targetNode = nodeService.findNode(batch.getNodeId(), true); + if (targetNode == null || !targetNode.isVersionGreaterThanOrEqualTo(3, 18)) { + return null; + } + return new StagedResourceETag(stagedResource.getGenerationTime(), stagedResource.getSize()); + } + protected String getBatchStatsColumns() { return StringUtils.join(new String[] { CsvConstants.STATS_COLUMNS, DataReaderStatistics.LOAD_FLAG, DataReaderStatistics.EXTRACT_COUNT, DataReaderStatistics.SENT_COUNT, DataReaderStatistics.LOAD_COUNT, DataReaderStatistics.LOAD_ID, @@ -2444,4 +2554,84 @@ public void release() { StagingFileLock fileLock; int referenceCount = 0; } + + protected static class StagedBatchTransferRequest { + private final ExtractMode mode; + private final BatchType batchType; + private final OutgoingBatch batch; + private final boolean isRetry; + private final IStagedResource stagedResource; + private final boolean isSuppressPreambleExtras; + + public StagedBatchTransferRequest(ExtractMode mode, BatchType batchType, OutgoingBatch batch, boolean isRetry, + IStagedResource stagedResource, boolean isSuppressPreambleExtras) { + this.mode = mode; + this.batchType = batchType; + this.batch = batch; + this.isRetry = isRetry; + this.stagedResource = stagedResource; + this.isSuppressPreambleExtras = isSuppressPreambleExtras; + } + + public ExtractMode getMode() { + return mode; + } + + public BatchType getBatchType() { + return batchType; + } + + public OutgoingBatch getBatch() { + return batch; + } + + public boolean isRetry() { + return isRetry; + } + + public IStagedResource getStagedResource() { + return stagedResource; + } + + public boolean isSuppressPreambleExtras() { + return isSuppressPreambleExtras; + } + } + + private static final class TransferProgress { + private final OutgoingBatch batch; + private final ExtractMode mode; + private final boolean isSuppressPreambleExtras; + private final boolean is39orNewer; + private final StagedResourceETag resumeEtag; + private final boolean isThrottled; + private final int bufferSize; + private final BigDecimal maxKBytesPerSec; + private final long totalBytes; + private final long startTime; + private long ts; + private long bts; + private long totalCharsRead; + private long totalBytesRead; + private long numBytesRead; + private long totalThrottleTime; + private boolean batchPreambleExtrasWritten; + private String prevBuffer = ""; + + private TransferProgress(StagedBatchTransferRequest request, boolean is39orNewer, StagedResourceETag resumeEtag, boolean isThrottled, + int bufferSize, BigDecimal maxKBytesPerSec, long totalBytes) { + this.batch = request.getBatch(); + this.mode = request.getMode(); + this.isSuppressPreambleExtras = request.isSuppressPreambleExtras(); + this.is39orNewer = is39orNewer; + this.resumeEtag = resumeEtag; + this.isThrottled = isThrottled; + this.bufferSize = bufferSize; + this.maxKBytesPerSec = maxKBytesPerSec; + this.totalBytes = totalBytes; + this.startTime = System.currentTimeMillis(); + this.ts = startTime; + this.bts = startTime; + } + } } diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataLoaderService.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataLoaderService.java index b7402ba3df..fcec16ecdd 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataLoaderService.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataLoaderService.java @@ -102,6 +102,8 @@ import org.jumpmind.symmetric.io.stage.IStagedResource.State; import org.jumpmind.symmetric.io.stage.SimpleStagingDataWriter; import org.jumpmind.symmetric.io.stage.StagingLowFreeSpace; +import org.jumpmind.symmetric.transport.http.IHttpResumeCache; +import org.jumpmind.symmetric.transport.http.ResumeCacheEntry; import org.jumpmind.symmetric.load.ConfigurationChangedDatabaseWriterFilter; import org.jumpmind.symmetric.load.DefaultDataLoaderFactory; import org.jumpmind.symmetric.load.DynamicDatabaseWriterFilter; @@ -270,6 +272,7 @@ public void loadDataFromPull(Node remote, RemoteNodeStatus status) throws IOExce String registrationUrl = parameterService.getRegistrationUrl(); IIncomingTransport transport = null; boolean isRegisterTransport = false; + ResumeCacheEntry confirmedResumeEntry = null; if (remote != null && localSecurity != null) { Map requestProperties = new HashMap(); NodeChannels suspendIgnoreChannels = configurationService @@ -279,8 +282,30 @@ public void loadDataFromPull(Node remote, RemoteNodeStatus status) throws IOExce requestProperties.put(WebConstants.IGNORED_CHANNELS, suspendIgnoreChannels.getIgnoreChannelsAsString(local.getNodeId())); requestProperties.put(WebConstants.CHANNEL_QUEUE, status.getQueue()); + IHttpResumeCache resumeCache = transportManager.getResumeCache(); + ResumeCacheEntry pendingResume = resumeCache != null ? resumeCache.getPendingForNode(remote.getNodeId(), status.getQueue()) : null; + if (pendingResume != null && pendingResume.isFileSync()) { + pendingResume = null; + } + if (pendingResume != null) { + requestProperties.put(WebConstants.HEADER_IF_ETAG, pendingResume.getEtag().toJson()); + requestProperties.put(WebConstants.HEADER_RANGE, "chars=" + pendingResume.getReceivedCount() + "-"); + } transport = transportManager.getPullTransport(remote, local, - localSecurity.getNodePassword(), requestProperties, registrationUrl); + localSecurity.getNodePassword(), requestProperties, registrationUrl, + pendingResume != null ? pendingResume.getBatchId() : null); + if (pendingResume != null) { + boolean serverResumed = transport.getHeaders().containsKey(WebConstants.HEADER_CONTENT_RANGE); + if (serverResumed) { + confirmedResumeEntry = pendingResume; + log.info("Resuming batch {} from node {}: server honored the resumed retry, appending to {} characters already received.", + pendingResume.getBatchId(), remote.getNodeId(), pendingResume.getReceivedCount()); + } else { + log.info("Resume of batch {} from node {} was not honored by the server (stale etag, resume disabled, or an older peer). " + + "Falling back to a full pull.", pendingResume.getBatchId(), remote.getNodeId()); + resumeCache.remove(remote.getNodeId(), pendingResume.getBatchId()); + } + } } else { List registrationListeners = extensionService.getExtensionPointList(INodeRegistrationListener.class); Map requestProps = new HashMap(); @@ -303,7 +328,7 @@ public void loadDataFromPull(Node remote, RemoteNodeStatus status) throws IOExce ProcessInfo transferInfo = statisticManager.newProcessInfo(new ProcessInfoKey(remote .getNodeId(), status.getQueue(), local.getNodeId(), PULL_JOB_TRANSFER)); try { - List list = loadDataFromTransport(transferInfo, remote, transport, null, status); + List list = loadDataFromTransport(transferInfo, remote, transport, null, status, confirmedResumeEntry); if (list.size() > 0) { transferInfo.setStatus(ProcessInfo.ProcessStatus.ACKING); status.updateIncomingStatus(list); @@ -363,7 +388,7 @@ public void loadDataFromPull(Node remote, RemoteNodeStatus status) throws IOExce } } - protected void updateBatchToSendCount(Node remote, IIncomingTransport transport) { + protected void updateBatchToSendCount(Node remote, IIncomingTransport transport) throws IOException { Map headers = transport.getHeaders(); if (headers != null && headers.containsKey(WebConstants.BATCH_TO_SEND_COUNT)) { Map queuesToBatchCounts = nodeCommunicationService.parseQueueToBatchCounts(headers.get(WebConstants.BATCH_TO_SEND_COUNT)); @@ -543,12 +568,19 @@ public List loadDataFromTransport(ProcessInfo processInfo, Node s return loadDataFromTransport(processInfo, sourceNode, transport, null, null); } + protected List loadDataFromTransport(final ProcessInfo transferInfo, + final Node sourceNode, IIncomingTransport transport, OutputStream out, RemoteNodeStatus status) throws IOException { + return loadDataFromTransport(transferInfo, sourceNode, transport, out, status, null); + } + /** * Load database from input stream and return a list of batch statuses. This is used for a pull request that responds with data, and the acknowledgment is - * sent later. + * sent later. {@code confirmedResumeEntry} is non-null only when the caller confirmed (via the response headers) that the server actually honored a + * requested resume of that one specific batch; it's {@code null} for every other call site, including a pull whose resume request was declined. */ protected List loadDataFromTransport(final ProcessInfo transferInfo, - final Node sourceNode, IIncomingTransport transport, OutputStream out, RemoteNodeStatus status) throws IOException { + final Node sourceNode, IIncomingTransport transport, OutputStream out, RemoteNodeStatus status, ResumeCacheEntry confirmedResumeEntry) + throws IOException { final ManageIncomingBatchListener listener = new ManageIncomingBatchListener(transferInfo, engine); final DataContext ctx = new DataContext(); Throwable error = null; @@ -582,8 +614,19 @@ protected List loadDataFromTransport(final ProcessInfo transferIn sourceNode.getNodeId(), listener, executor); SimpleStagingDataWriter stageWriter = null; try { - stageWriter = new SimpleStagingDataWriter(transferInfo, transport.openReader(), engine, Constants.STAGING_CATEGORY_INCOMING, - memoryThresholdInBytes, BatchType.LOAD, sourceNode.getNodeId(), targetNodeId, ctx, loadListener); + stageWriter = SimpleStagingDataWriter.builder() + .processInfo(transferInfo) + .reader(transport.openReader()) + .engine(engine) + .category(Constants.STAGING_CATEGORY_INCOMING) + .memoryThresholdInBytes(memoryThresholdInBytes) + .batchType(BatchType.LOAD) + .sourceNodeId(sourceNode.getNodeId()) + .targetNodeId(targetNodeId) + .context(ctx) + .resumeEntry(confirmedResumeEntry) + .listeners(loadListener) + .build(); notifyQueuesReady(status, transport); stageWriter.process(); } finally { @@ -668,7 +711,7 @@ protected IDataWriter chooseDataWriter(Batch batch) { return batchesProcessed; } - private void notifyQueuesReady(RemoteNodeStatus status, IIncomingTransport transport) { + private void notifyQueuesReady(RemoteNodeStatus status, IIncomingTransport transport) throws IOException { if (parameterService.is(ParameterConstants.SYNC_USE_READY_QUEUES) && configurationService.getQueues(false).size() > 1 && !parameterService.is(ParameterConstants.ROUTE_ON_EXTRACT) && status != null && Constants.QUEUE_DEFAULT.equals(status.getQueue())) { Map headers = transport.getHeaders(); diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/FileSyncService.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/FileSyncService.java index 08371b01a4..bc535918f2 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/FileSyncService.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/FileSyncService.java @@ -30,18 +30,24 @@ import java.nio.file.Path; import java.sql.Types; import java.util.ArrayList; +import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeSet; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.io.filefilter.DirectoryFileFilter; +import org.apache.commons.io.input.BoundedInputStream; import org.apache.commons.io.monitor.FileAlterationObserver; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; import org.jumpmind.db.model.Relation; import org.jumpmind.db.sql.ISqlReadCursor; import org.jumpmind.db.sql.ISqlRowMapper; @@ -58,6 +64,8 @@ import org.jumpmind.symmetric.common.TableConstants; import org.jumpmind.symmetric.file.DirectorySnapshot; import org.jumpmind.symmetric.file.FileConflictException; +import org.jumpmind.symmetric.file.FileSyncBatchEnvelope; +import org.jumpmind.symmetric.file.FileSyncPullResult; import org.jumpmind.symmetric.file.FileSyncZipDataWriter; import org.jumpmind.symmetric.file.FileTriggerFileModifiedListener; import org.jumpmind.symmetric.file.FileTriggerFileModifiedListener.FileModifiedCallback; @@ -69,6 +77,7 @@ import org.jumpmind.symmetric.io.stage.IStagedResource; import org.jumpmind.symmetric.io.stage.IStagedResource.State; import org.jumpmind.symmetric.io.stage.IStagingManager; +import org.jumpmind.symmetric.io.stage.StagedResourceETag; import org.jumpmind.symmetric.model.AbstractBatch.Status; import org.jumpmind.symmetric.model.BatchAck; import org.jumpmind.symmetric.model.Channel; @@ -103,6 +112,9 @@ import org.jumpmind.symmetric.transport.NoContentException; import org.jumpmind.symmetric.transport.file.FileIncomingTransport; import org.jumpmind.symmetric.transport.file.FileOutgoingTransport; +import org.jumpmind.symmetric.transport.http.IHttpResumeCache; +import org.jumpmind.symmetric.transport.http.ResumeCacheEntry; +import org.jumpmind.symmetric.web.WebConstants; import org.jumpmind.util.AppUtils; import org.jumpmind.util.ExceptionUtils; @@ -112,6 +124,8 @@ public class FileSyncService extends AbstractOfflineDetectorService implements IFileSyncService, INodeCommunicationExecutor { + private static final Pattern RANGE_PATTERN = Pattern.compile("bytes=(\\d{1,18})-"); + private static final String FILESYNC_STAGING_SUFFIX = "_filesync"; private ISymmetricEngine engine; private Date lastUpdateTime; private ICacheManager cacheManager; @@ -654,11 +668,53 @@ synchronized public RemoteNodeStatuses pushFilesToNodes(boolean force) { @Override public Object[] getStagingPathComponents(OutgoingBatch fileSyncBatch) { StringBuilder zipName = new StringBuilder(32); - zipName.append(StringUtils.leftPad(String.valueOf(fileSyncBatch.getBatchId()), 10, "0")).append("_filesync"); + zipName.append(StringUtils.leftPad(String.valueOf(fileSyncBatch.getBatchId()), 10, "0")).append(FILESYNC_STAGING_SUFFIX); return new String[] { Constants.STAGING_CATEGORY_OUTGOING, Batch.getStagedLocation(fileSyncBatch.isCommonFlag(), fileSyncBatch.getNodeId(), fileSyncBatch.getBatchId()), zipName.toString() }; } + private void markBatchesTransferring(ProcessInfo processInfo, List processedBatches) { + processInfo.setStatus(ProcessInfo.ProcessStatus.TRANSFERRING); + for (OutgoingBatch outgoingBatch : processedBatches) { + outgoingBatch.setStatus(Status.SE); + } + engine.getOutgoingBatchService().updateOutgoingBatches(processedBatches); + } + + private void markBatchesLoaded(List batchesToProcess) { + for (int i = 0; i < batchesToProcess.size(); i++) { + batchesToProcess.get(i).setStatus(Status.LD); + } + engine.getOutgoingBatchService().updateOutgoingBatches(batchesToProcess); + } + + private void handleExtractionError(ProcessInfo processInfo, RuntimeException e, OutgoingBatch currentBatch) { + if (currentBatch != null) { + if (processInfo.getStatus() == ProcessInfo.ProcessStatus.TRANSFERRING) { + engine.getStatisticManager().incrementDataSentErrors(currentBatch.getChannelId(), 1); + } else { + engine.getStatisticManager().incrementDataExtractedErrors(currentBatch.getChannelId(), 1); + } + currentBatch.setSqlMessage(ExceptionUtils.getRootMessage(e)); + currentBatch.revertStatsOnError(); + if (currentBatch.getStatus() != Status.IG) { + currentBatch.setStatus(Status.ER); + } + currentBatch.setErrorFlag(true); + engine.getOutgoingBatchService().updateOutgoingBatch(currentBatch); + if (isStreamClosedByClient(e)) { + if (log.isWarnEnabled()) { + log.warn("Failed to extract file sync batch {}. The stream was closed by the client. The error was: {}", + currentBatch, ExceptionUtils.getRootMessage(e)); + } + } else { + log.error("Failed to extract file sync batch " + currentBatch, e); + } + } else { + log.error("Could not log the outgoing batch status because the batch was null", e); + } + } + @Override public List sendFiles(ProcessInfo processInfo, Node targetNode, IOutgoingTransport outgoingTransport) { @@ -719,11 +775,7 @@ public List sendFiles(ProcessInfo processInfo, Node targetNode, dataWriter.finish(); } } - processInfo.setStatus(ProcessInfo.ProcessStatus.TRANSFERRING); - for (OutgoingBatch outgoingBatch : processedBatches) { - outgoingBatch.setStatus(Status.SE); - } - engine.getOutgoingBatchService().updateOutgoingBatches(processedBatches); + markBatchesTransferring(processInfo, processedBatches); try { if (stagedResource != null && stagedResource.exists()) { InputStream is = stagedResource.getInputStream(); @@ -738,10 +790,7 @@ public List sendFiles(ProcessInfo processInfo, Node targetNode, log.error("Missing staged ZIP file for target node {}: {}", targetNode, stagedResource == null ? "" : stagedResource); } - for (int i = 0; i < processedBatches.size(); i++) { - processedBatches.get(i).setStatus(Status.LD); - } - engine.getOutgoingBatchService().updateOutgoingBatches(processedBatches); + markBatchesLoaded(processedBatches); } finally { if (stagedResource != null) { stagedResource.close(); @@ -751,36 +800,230 @@ public List sendFiles(ProcessInfo processInfo, Node targetNode, if (stagedResource == previouslyStagedResource) { // on error, don't let the load extract be deleted. stagedResource = null; } - if (currentBatch != null) { - if (processInfo.getStatus() == ProcessInfo.ProcessStatus.TRANSFERRING) { - engine.getStatisticManager().incrementDataSentErrors(currentBatch.getChannelId(), 1); + handleExtractionError(processInfo, e, currentBatch); + throw e; + } finally { + if (stagedResource != null && parameterService.is(ParameterConstants.FILE_SYNC_DELETE_ZIP_FILE_AFTER_SYNC)) { + stagedResource.delete(); + } + } + return processedBatches; + } + + /** + * Same overall shape as {@link #sendFiles(ProcessInfo, Node, IOutgoingTransport)}, but used only by the pull path: honors a resume request for one specific + * batch, and (for 3.18+ peers, with resume enabled) restructures extraction so each batch gets its own independently-staged, independently-finished zip + * instead of sharing one growing zip across the whole loop — a prerequisite for being able to resume any one batch on its own. Bundling multiple such + * batches into a single response requires the peer to understand the {@code FileSync-Format} envelope; older or unrecognized peers get exactly today's + * behavior, one complete zip containing a single batch per response. + *

+ * Performs no network writes - only extraction/staging and header/format decisions - so the caller can set response headers on the returned result before + * calling {@link #writeFilesForPull(FileSyncPullResult, IOutgoingTransport)}. + */ + @Override + public FileSyncPullResult prepareFilesForPull(ProcessInfo processInfo, Node targetNode, String batchIdParam, + String ifETagHeader, String rangeHeader) { + boolean isResumeEnabled = parameterService.is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED); + if (StringUtils.isNotBlank(batchIdParam) && isResumeEnabled) { + FileSyncPullResult resumeResult = prepareResumedBatch(processInfo, targetNode, batchIdParam, ifETagHeader, rangeHeader); + if (resumeResult != null) { + return resumeResult; + } + } + boolean useEnvelope = isResumeEnabled && targetNode != null && targetNode.isVersionGreaterThanOrEqualTo(3, 18); + List batchesToProcess = getBatchesToProcess(targetNode); + if (batchesToProcess.isEmpty()) { + return FileSyncPullResult.builder().batches(batchesToProcess).allRequestedBatches(batchesToProcess) + .stagedResources(new ArrayList()).envelopeFormatUsed(false).build(); + } + long maxBytesToSync = parameterService.getLong(ParameterConstants.TRANSPORT_MAX_BYTES_TO_SYNC); + int compressionLevel = parameterService.getInt(ParameterConstants.FILE_SYNC_COMPRESSION_LEVEL); + List processedBatches = new ArrayList(); + List processedResources = new ArrayList(); + OutgoingBatch currentBatch = null; + try { + long syncedBytes = 0; + boolean shouldStop = false; + for (int i = 0; i < batchesToProcess.size() && !shouldStop; i++) { + currentBatch = batchesToProcess.get(i); + IStagedResource previouslyStagedResource = getStagedResource(currentBatch); + if (isWaitForExtractionRequired(currentBatch, previouslyStagedResource) + || isFlushBatchesRequired(currentBatch, processedBatches, previouslyStagedResource)) { + shouldStop = true; } else { - engine.getStatisticManager().incrementDataExtractedErrors(currentBatch.getChannelId(), 1); - } - currentBatch.setSqlMessage(ExceptionUtils.getRootMessage(e)); - currentBatch.revertStatsOnError(); - if (currentBatch.getStatus() != Status.IG) { - currentBatch.setStatus(Status.ER); + IStagedResource stagedResource = extractOrReuseStagedBatch(processInfo, targetNode, currentBatch, + previouslyStagedResource, maxBytesToSync, compressionLevel); + processedBatches.add(currentBatch); + processedResources.add(stagedResource); + syncedBytes += stagedResource.getSize(); + processInfo.incrementBatchCount(); + processInfo.setCurrentBatchId(currentBatch.getBatchId()); + log.debug("Processed file sync batch {}. syncedBytes={}, maxBytesToSync={}", currentBatch, syncedBytes, maxBytesToSync); + shouldStop = !useEnvelope || syncedBytes > maxBytesToSync; } - currentBatch.setErrorFlag(true); - engine.getOutgoingBatchService().updateOutgoingBatch(currentBatch); - if (isStreamClosedByClient(e)) { - log.warn( - "Failed to extract file sync batch {}. The stream was closed by the client. The error was: {}", - currentBatch, ExceptionUtils.getRootMessage(e)); - } else { - log.error("Failed to extract file sync batch " + currentBatch, e); + } + markBatchesTransferring(processInfo, processedBatches); + } catch (RuntimeException e) { + handleExtractionError(processInfo, e, currentBatch); + closeStagedResources(processedResources); + throw e; + } + return FileSyncPullResult.builder().batches(processedBatches).allRequestedBatches(batchesToProcess) + .stagedResources(processedResources).envelopeFormatUsed(useEnvelope).build(); + } + + private IStagedResource extractOrReuseStagedBatch(ProcessInfo processInfo, Node targetNode, OutgoingBatch currentBatch, + IStagedResource previouslyStagedResource, long maxBytesToSync, int compressionLevel) { + if (previouslyStagedResource != null) { + log.debug("Using existing extraction for file sync batch {}", currentBatch.getNodeBatchId()); + return previouslyStagedResource; + } + IStagedResource stagedResource = engine.getStagingManager().create(getStagingPathComponents(currentBatch)); + FileSyncZipDataWriter dataWriter = new FileSyncZipDataWriter(maxBytesToSync, compressionLevel, this, + engine.getNodeService(), stagedResource, engine.getExtensionService(), engine.getConfigurationService()); + try { + log.debug("Extracting batch {} for filesync.", currentBatch.getNodeBatchId()); + ((DataExtractorService) engine.getDataExtractorService()).extractOutgoingBatch( + processInfo, targetNode, dataWriter, currentBatch, false, true, + DataExtractorService.ExtractMode.FOR_SYM_CLIENT, null); + } finally { + dataWriter.finish(); + } + return stagedResource; + } + + /** + * Streams the bytes described by a {@link FileSyncPullResult} previously returned from + * {@link #prepareFilesForPull(ProcessInfo, Node, String, String, String)}. Must be called only after the caller has finished setting response + * headers/status, since writing to {@code outgoingTransport} commits the response. + */ + @Override + public void writeFilesForPull(ProcessInfo processInfo, FileSyncPullResult result, IOutgoingTransport outgoingTransport) { + if (result.getResumeEtag() != null) { + writeResumedBatch(result, outgoingTransport); + } else { + writeNormalBatches(processInfo, result, outgoingTransport); + } + } + + private void writeResumedBatch(FileSyncPullResult result, IOutgoingTransport outgoingTransport) { + List stagedResources = result.getStagedResources(); + if (stagedResources == null || stagedResources.isEmpty()) { + return; + } + IStagedResource stagedResource = stagedResources.get(0); + long skipCount = result.getSkipCount(); + try { + OutputStream os = outgoingTransport.openStream(); + try (InputStream is = stagedResource.getInputStream()) { + if (skipCount > 0) { + IOUtils.skipFully(is, skipCount); } - } else { - log.error("Could not log the outgoing batch status because the batch was null", e); + IOUtils.copy(is, os); } + os.flush(); + } catch (IOException e) { + throw new IoException(e); + } + log.debug("Served {} file sync pull for batch {} to node {} ({} of {} skipped)", result.isPartialContent() ? "resumed" : "full", + result.getBatches().get(0).getBatchId(), result.getBatches().get(0).getNodeId(), skipCount, result.getTotalSize()); + } + + private void writeNormalBatches(ProcessInfo processInfo, FileSyncPullResult result, IOutgoingTransport outgoingTransport) { + List processedBatches = result.getBatches(); + List processedResources = result.getStagedResources(); + OutgoingBatch lastBatch = processedBatches.isEmpty() ? null : processedBatches.get(processedBatches.size() - 1); + try { + writeBatchesToStream(result, outgoingTransport, processedBatches, processedResources); + } catch (RuntimeException e) { + handleExtractionError(processInfo, e, lastBatch); throw e; } finally { - if (stagedResource != null && parameterService.is(ParameterConstants.FILE_SYNC_DELETE_ZIP_FILE_AFTER_SYNC)) { - stagedResource.delete(); + closeStagedResources(processedResources); + } + } + + private void writeBatchesToStream(FileSyncPullResult result, IOutgoingTransport outgoingTransport, + List processedBatches, List processedResources) { + try { + OutputStream os = outgoingTransport.openStream(); + if (result.isEnvelopeFormatUsed()) { + for (int i = 0; i < processedBatches.size(); i++) { + IStagedResource resource = processedResources.get(i); + StagedResourceETag etag = new StagedResourceETag(resource.getGenerationTime(), resource.getSize()); + FileSyncBatchEnvelope.writeHeader(os, processedBatches.get(i).getBatchId(), resource.getSize(), etag); + try (InputStream is = resource.getInputStream()) { + IOUtils.copy(is, os); + } + } + } else if (!processedResources.isEmpty()) { + try (InputStream is = processedResources.get(0).getInputStream()) { + IOUtils.copy(is, os); + } } + os.flush(); + markBatchesLoaded(result.getAllRequestedBatches()); + } catch (IOException e) { + throw new IoException(e); } - return processedBatches; + } + + private void closeStagedResources(List resources) { + boolean deleteAfterSync = parameterService.is(ParameterConstants.FILE_SYNC_DELETE_ZIP_FILE_AFTER_SYNC); + for (IStagedResource resource : resources) { + resource.close(); + if (deleteAfterSync) { + resource.delete(); + } + } + } + + /** + * Prepares to serve a single, previously-interrupted batch pull directly from its own staged zip, instead of the normal multi-batch path. Returns + * {@code null} whenever no valid, fully-staged, file-backed resource exists for the requested batch, so the caller can fall through to a normal pull + * unchanged. + */ + private FileSyncPullResult prepareResumedBatch(ProcessInfo processInfo, Node targetNode, String batchIdParam, + String ifETagHeader, String rangeHeader) { + long batchId = NumberUtils.toLong(batchIdParam, -1L); + if (batchId < 0 || targetNode == null) { + return null; + } + OutgoingBatch batch = engine.getOutgoingBatchService().findOutgoingBatch(batchId, targetNode.getNodeId()); + IStagedResource stagedResource = getStagedResource(batch); + if (batch == null || stagedResource == null || stagedResource.getState() != State.DONE || !stagedResource.isFileResource()) { + log.debug("Resume requested for file sync batch {} from node {}, but no resumable staged resource was found. Falling back to a full pull.", + batchId, targetNode.getNodeId()); + return null; + } + long totalSize = stagedResource.getSize(); + StagedResourceETag etag = new StagedResourceETag(stagedResource.getGenerationTime(), totalSize); + StagedResourceETag requestedETag = StagedResourceETag.fromJson(ifETagHeader); + Long requestedSkipCount = parseRangeSkipCount(rangeHeader); + boolean isPartial = etag.equals(requestedETag) && requestedSkipCount != null && requestedSkipCount >= 0 + && requestedSkipCount < totalSize; + long skipCount = isPartial ? requestedSkipCount : 0; + processInfo.incrementBatchCount(); + processInfo.setCurrentBatchId(batch.getBatchId()); + List batches = new ArrayList(); + batches.add(batch); + List stagedResources = new ArrayList(); + stagedResources.add(stagedResource); + log.debug("Prepared {} file sync pull for batch {} to node {} ({} of {} skipped)", isPartial ? "resumed" : "full", batchId, + targetNode.getNodeId(), skipCount, totalSize); + return FileSyncPullResult.builder().batches(batches).stagedResources(stagedResources).partialContent(isPartial) + .resumeEtag(etag).totalSize(totalSize).skipCount(skipCount).build(); + } + + private Long parseRangeSkipCount(String rangeHeader) { + if (StringUtils.isBlank(rangeHeader)) { + return null; + } + Matcher matcher = RANGE_PATTERN.matcher(rangeHeader.trim()); + if (!matcher.matches()) { + return null; + } + return Long.parseLong(matcher.group(1)); } private boolean isFlushBatchesRequired(OutgoingBatch currentBatch, List processedBatches, IStagedResource previouslyStagedResource) { @@ -917,6 +1160,9 @@ public void loadFilesFromPush(String nodeId, InputStream in, OutputStream out) { } protected IStagedResource getStagedResource(OutgoingBatch currentBatch) { + if (currentBatch == null) { + return null; + } IStagedResource stagedResource = engine.getStagingManager().find(getStagingPathComponents(currentBatch)); if (stagedResource != null && stagedResource.getState() == State.DONE) { return stagedResource; @@ -1018,6 +1264,25 @@ protected void pushFilesToNode(NodeCommunication nodeCommunication, RemoteNodeSt protected List processZip(InputStream is, String sourceNodeId, ProcessInfo processInfo) throws IOException { + File unzipDir = prepareUnzipDir(sourceNodeId); + try { + AppUtils.unzip(is, unzipDir); + } catch (IoException ex) { + if (ex.toString().contains("EOFException")) { // This happens on Android, when there is an empty zip. + // log.debug("Caught exception while unzipping.", ex); + } else { + throw ex; + } + } + return processUnzippedBatches(unzipDir, sourceNodeId, processInfo); + } + + /** + * Resolves (and freshly (re)creates) the local directory that a pull response's zip content is unzipped into, shared by the legacy whole-response unzip + * path and the {@code FileSync-Format} envelope path so that both populate the same {@code /...} layout for {@link #processUnzippedBatches} to + * discover. + */ + private File prepareUnzipDir(String sourceNodeId) throws IOException { Path tempDirPath = Path.of(parameterService.getTempDirectory()).toAbsolutePath().normalize(); Path unzipPath = tempDirPath.resolve(String.format( "filesync_incoming/%s/%s", engine.getNodeService().findIdentityNodeId(), @@ -1028,15 +1293,121 @@ protected List processZip(InputStream is, String sourceNodeId, File unzipDir = unzipPath.toFile(); FileUtils.deleteDirectory(unzipDir); Files.createDirectories(unzipPath); + return unzipDir; + } + + /** + * Reads each batch's independently-staged, length-verified zip from the {@code FileSync-Format} envelope and unzips it individually, registering an + * interrupted batch for resume rather than leaving a half-written local copy behind. Falls through to {@link #processUnzippedBatches} once every batch + * declared in the envelope has been unzipped. + */ + protected List processEnvelopedZip(InputStream is, String sourceNodeId, ProcessInfo processInfo) throws IOException { + File unzipDir = prepareUnzipDir(sourceNodeId); + IStagingManager stagingManager = engine.getStagingManager(); + IHttpResumeCache resumeCache = engine.getTransportManager() != null ? engine.getTransportManager().getResumeCache() : null; + boolean isResumeEnabled = parameterService.is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED); + FileSyncBatchEnvelope header; + while ((header = FileSyncBatchEnvelope.readHeader(is)) != null) { + IStagedResource localResource = stagingManager.create(Constants.STAGING_CATEGORY_INCOMING, sourceNodeId, + header.getBatchId() + FILESYNC_STAGING_SUFFIX); + try { + OutputStream out = localResource.getOutputStream(); + long received = IOUtils.copyLarge(BoundedInputStream.builder().setInputStream(is).setMaxCount(header.getLength()).get(), out); + out.flush(); + if (received != header.getLength()) { + throw new IOException("Expected " + header.getLength() + " bytes for file sync batch " + header.getBatchId() + + " from node " + sourceNodeId + " but only received " + received); + } + localResource.close(); + localResource.setState(State.DONE); + } catch (IOException e) { + localResource.close(); + if (isResumeEnabled && resumeCache != null) { + resumeCache.put(sourceNodeId, header.getBatchId(), ResumeCacheEntry.builder() + .nodeId(sourceNodeId) + .batchId(header.getBatchId()) + .etag(header.getEtag()) + .receivedCount(localResource.getSize()) + .fileSync(true) + .cachedAtTime(System.currentTimeMillis()) + .queue(processInfo.getQueue()) + .build()); + log.info("Preserving partially-received file sync batch {} from node {} for a resumed retry ({} of {} bytes received).", + header.getBatchId(), sourceNodeId, localResource.getSize(), header.getLength()); + } else { + localResource.delete(); + } + throw e; + } + try (InputStream zipIs = localResource.getInputStream()) { + AppUtils.unzip(zipIs, unzipDir); + } finally { + localResource.delete(); + } + if (resumeCache != null) { + resumeCache.remove(sourceNodeId, header.getBatchId()); + } + } + return processUnzippedBatches(unzipDir, sourceNodeId, processInfo); + } + + /** + * Appends a confirmed resumed response — the raw continuation bytes of one specific, previously-partial batch's zip, with no envelope framing since only + * one batch is involved — to the local partial copy left behind by {@link #processEnvelopedZip}'s registration, then unzips and processes it like any other + * batch. + */ + protected List resumePartialBatch(InputStream is, String sourceNodeId, ProcessInfo processInfo, + ResumeCacheEntry pendingResume) throws IOException { + long batchId = pendingResume.getBatchId(); + log.info("Resuming file sync batch {} from node {}: server honored the resumed retry, appending to {} bytes already received.", + batchId, sourceNodeId, pendingResume.getReceivedCount()); + IStagingManager stagingManager = engine.getStagingManager(); + IHttpResumeCache resumeCache = engine.getTransportManager() != null ? engine.getTransportManager().getResumeCache() : null; + IStagedResource localResource = stagingManager.find(Constants.STAGING_CATEGORY_INCOMING, sourceNodeId, + batchId + FILESYNC_STAGING_SUFFIX); + if (localResource == null || localResource.getState() != State.CREATE) { + log.warn("Resume requested for file sync batch {} from node {}, but the local partial staged resource was missing or already " + + "finalized ({}). This pull cannot complete that batch; a subsequent pull will retry it in full.", + batchId, sourceNodeId, localResource == null ? "not found" : localResource.getState()); + if (resumeCache != null) { + resumeCache.remove(sourceNodeId, batchId); + } + return new ArrayList(); + } try { - AppUtils.unzip(is, unzipDir); - } catch (IoException ex) { - if (ex.toString().contains("EOFException")) { // This happens on Android, when there is an empty zip. - // log.debug("Caught exception while unzipping.", ex); - } else { - throw ex; + OutputStream out = localResource.getOutputStream(true); + IOUtils.copy(is, out); + out.flush(); + localResource.close(); + localResource.setState(State.DONE); + } catch (IOException e) { + localResource.close(); + if (resumeCache != null) { + resumeCache.put(sourceNodeId, batchId, ResumeCacheEntry.builder() + .nodeId(sourceNodeId) + .batchId(batchId) + .etag(pendingResume.getEtag()) + .receivedCount(localResource.getSize()) + .fileSync(true) + .cachedAtTime(System.currentTimeMillis()) + .queue(pendingResume.getQueue()) + .build()); } + throw e; + } + File unzipDir = prepareUnzipDir(sourceNodeId); + try (InputStream zipIs = localResource.getInputStream()) { + AppUtils.unzip(zipIs, unzipDir); + } finally { + localResource.delete(); } + if (resumeCache != null) { + resumeCache.remove(sourceNodeId, batchId); + } + return processUnzippedBatches(unzipDir, sourceNodeId, processInfo); + } + + private List processUnzippedBatches(File unzipDir, String sourceNodeId, ProcessInfo processInfo) throws IOException { Set batchIds = new TreeSet(); String[] files = unzipDir.list(DirectoryFileFilter.INSTANCE); if (files != null) { @@ -1211,24 +1582,24 @@ protected void pullFilesFromNode(NodeCommunication nodeCommunication, RemoteNode Node identity, NodeSecurity security) { IIncomingTransport transport = null; ProcessInfo processInfo = engine.getStatisticManager().newProcessInfo( - new ProcessInfoKey(nodeCommunication.getNodeId(), identity.getNodeId(), + new ProcessInfoKey(nodeCommunication.getNodeId(), status.getQueue(), identity.getNodeId(), ProcessType.FILE_SYNC_PULL_JOB)); try { processInfo.setStatus(ProcessInfo.ProcessStatus.TRANSFERRING); ITransportManager transportManager; if (!engine.getParameterService().is(ParameterConstants.NODE_OFFLINE)) { transportManager = engine.getTransportManager(); - transport = transportManager.getFilePullTransport( - nodeCommunication.getNode(), identity, security.getNodePassword(), null, - parameterService.getRegistrationUrl()); } else { transportManager = ((AbstractSymmetricEngine) engine).getOfflineTransportManager(); - transport = transportManager.getFilePullTransport( - nodeCommunication.getNode(), identity, security.getNodePassword(), null, - parameterService.getRegistrationUrl()); } - List batchesProcessed = processZip(transport.openStream(), - nodeCommunication.getNodeId(), processInfo); + IHttpResumeCache resumeCache = transportManager.getResumeCache(); + ResumeCacheEntry pendingResume = resumeCache != null + ? resumeCache.getPendingFileSyncEntryForNode(nodeCommunication.getNodeId()) + : null; + transport = transportManager.getFilePullTransport( + nodeCommunication.getNode(), identity, security.getNodePassword(), buildResumeRequestProperties(pendingResume), + parameterService.getRegistrationUrl(), pendingResume != null ? pendingResume.getBatchId() : null); + List batchesProcessed = receiveFileSyncBatches(transport, nodeCommunication, processInfo, resumeCache, pendingResume); if (batchesProcessed.size() > 0) { processInfo.setStatus(ProcessInfo.ProcessStatus.ACKING); status.updateIncomingStatus(batchesProcessed); @@ -1261,6 +1632,32 @@ protected void pullFilesFromNode(NodeCommunication nodeCommunication, RemoteNode } } + private Map buildResumeRequestProperties(ResumeCacheEntry pendingResume) { + if (pendingResume == null) { + return Collections.emptyMap(); + } + Map requestProperties = new HashMap(); + requestProperties.put(WebConstants.HEADER_IF_ETAG, pendingResume.getEtag().toJson()); + requestProperties.put(WebConstants.HEADER_RANGE, "bytes=" + pendingResume.getReceivedCount() + "-"); + return requestProperties; + } + + private List receiveFileSyncBatches(IIncomingTransport transport, NodeCommunication nodeCommunication, + ProcessInfo processInfo, IHttpResumeCache resumeCache, ResumeCacheEntry pendingResume) throws IOException { + if (pendingResume != null && transport.getHeaders().containsKey(WebConstants.HEADER_CONTENT_RANGE)) { + return resumePartialBatch(transport.openStream(), nodeCommunication.getNodeId(), processInfo, pendingResume); + } + if (pendingResume != null) { + log.info("Resume of file sync batch {} from node {} was not honored by the server (stale etag, resume disabled, " + + "or an older peer). Falling back to a full pull.", pendingResume.getBatchId(), nodeCommunication.getNodeId()); + resumeCache.remove(nodeCommunication.getNodeId(), pendingResume.getBatchId()); + } + if (transport.getHeaders().containsKey(WebConstants.HEADER_FILESYNC_FORMAT)) { + return processEnvelopedZip(transport.openStream(), nodeCommunication.getNodeId(), processInfo); + } + return processZip(transport.openStream(), nodeCommunication.getNodeId(), processInfo); + } + protected RemoteNodeStatuses queueJob(boolean force, long minimumPeriodMs, String clusterLock, CommunicationType type) { final RemoteNodeStatuses statuses = new RemoteNodeStatuses(engine.getConfigurationService().getChannels(false)); diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/HybridTransportManager.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/HybridTransportManager.java index d6cb98b6f1..bfcd6a27f5 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/HybridTransportManager.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/HybridTransportManager.java @@ -13,6 +13,7 @@ import org.jumpmind.symmetric.model.Node; import org.jumpmind.symmetric.service.IParameterService; import org.jumpmind.symmetric.transport.http.HttpTransportManager; +import org.jumpmind.symmetric.transport.http.IHttpResumeCache; import org.jumpmind.symmetric.transport.internal.InternalTransportManager; public class HybridTransportManager implements ITransportManager { @@ -55,6 +56,12 @@ public IIncomingTransport getFilePullTransport(Node remote, Node local, String s return getTransport(remote).getFilePullTransport(remote, local, securityToken, requestProperties, registrationUrl); } + @Override + public IIncomingTransport getFilePullTransport(Node remote, Node local, String securityToken, + Map requestProperties, String registrationUrl, Long resumeBatchId) throws IOException { + return getTransport(remote).getFilePullTransport(remote, local, securityToken, requestProperties, registrationUrl, resumeBatchId); + } + @Override public IOutgoingWithResponseTransport getFilePushTransport(Node remote, Node local, String securityToken, String registrationUrl) throws IOException { @@ -67,6 +74,26 @@ public IIncomingTransport getPullTransport(Node remote, Node local, String secur return getTransport(remote).getPullTransport(remote, local, securityToken, requestProperties, registrationUrl); } + /** + * Delegates to whichever underlying transport is active for this node, so a resumed pull still reaches {@link HttpTransportManager}'s real resume logic + * when hybrid mode has selected HTTP; the internal transport has no concept of resume and ignores the extra parameter via the {@code ITransportManager} + * default. + */ + @Override + public IIncomingTransport getPullTransport(Node remote, Node local, String securityToken, + Map requestProperties, String registrationUrl, Long resumeBatchId) throws IOException { + return getTransport(remote).getPullTransport(remote, local, securityToken, requestProperties, registrationUrl, resumeBatchId); + } + + /** + * The internal (same-JVM) transport never needs real HTTP resume, so it's safe to always resolve resume state through the one {@link HttpTransportManager} + * this hybrid manager holds, regardless of which transport ends up handling any particular pull. + */ + @Override + public IHttpResumeCache getResumeCache() { + return httpTransport.getResumeCache(); + } + @Override public IIncomingTransport getPingTransport(Node remote, Node local, String registrationUrl) throws IOException { return getTransport(remote).getPingTransport(remote, local, registrationUrl); diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/IIncomingTransport.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/IIncomingTransport.java index 2a1b7be10a..f78132a901 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/IIncomingTransport.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/IIncomingTransport.java @@ -38,5 +38,10 @@ public interface IIncomingTransport { public String getUrl(); - public Map getHeaders(); + /** + * @throws IOException + * if the underlying connection could not be established - callers must not treat this as a definitive "no such header" result (e.g. a resume + * request that hasn't been honored), since that would misclassify a transient connectivity failure as a real server response. + */ + public Map getHeaders() throws IOException; } \ No newline at end of file diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/ITransportManager.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/ITransportManager.java index 7e7ffc2351..a291d313d7 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/ITransportManager.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/ITransportManager.java @@ -29,6 +29,7 @@ import org.jumpmind.symmetric.model.BatchAck; import org.jumpmind.symmetric.model.IncomingBatch; import org.jumpmind.symmetric.model.Node; +import org.jumpmind.symmetric.transport.http.IHttpResumeCache; public interface ITransportManager { public int sendAcknowledgement(Node remote, List list, Node local, String securityToken, String registrationUrl) throws IOException; @@ -43,12 +44,39 @@ public int sendAcknowledgement(Node remote, List list, Node local public IIncomingTransport getFilePullTransport(Node remote, Node local, String securityToken, Map requestProperties, String registrationUrl) throws IOException; + /** + * Same as the 5-arg {@code getFilePullTransport}, but additionally requests a resumed pull of one specific previously-interrupted file sync batch when + * {@code resumeBatchId} is non-null. Implementations that don't support resume may ignore {@code resumeBatchId} and delegate to the 5-arg overload. + */ + default IIncomingTransport getFilePullTransport(Node remote, Node local, String securityToken, + Map requestProperties, String registrationUrl, Long resumeBatchId) throws IOException { + return getFilePullTransport(remote, local, securityToken, requestProperties, registrationUrl); + } + public IOutgoingWithResponseTransport getFilePushTransport(Node remote, Node local, String securityToken, String registrationUrl) throws IOException; public IIncomingTransport getPullTransport(Node remote, Node local, String securityToken, Map requestProperties, String registrationUrl) throws IOException; + /** + * Same as the 5-arg {@code getPullTransport}, but additionally requests a resumed pull of one specific previously-interrupted batch when + * {@code resumeBatchId} is non-null. Resume is an HTTP-specific mechanism; implementations that don't support it may ignore {@code resumeBatchId} and + * delegate to the 5-arg overload. + */ + default IIncomingTransport getPullTransport(Node remote, Node local, String securityToken, Map requestProperties, + String registrationUrl, Long resumeBatchId) throws IOException { + return getPullTransport(remote, local, securityToken, requestProperties, registrationUrl); + } + + /** + * @return the resume cache backing this transport manager's pull requests, or {@code null} if this transport doesn't support resume (resume is + * HTTP-specific) + */ + default IHttpResumeCache getResumeCache() { + return null; + } + public IIncomingTransport getPingTransport(Node remote, Node local, String registrationUrl) throws IOException; public IOutgoingWithResponseTransport getPushTransport(Node remote, Node local, String securityToken, String registrationUrl) throws IOException; diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/file/FileTransportManager.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/file/FileTransportManager.java index 0451ea64c3..1ee5acfa4c 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/file/FileTransportManager.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/file/FileTransportManager.java @@ -64,6 +64,16 @@ public IIncomingTransport getFilePullTransport(Node remote, Node local, String s return getPullTransport(remote, local, securityToken, requestProperties, registrationUrl); } + /** + * File-based transport has no concept of a resumed pull; ignore {@code resumeBatchId} rather than inheriting {@link HttpTransportManager}'s HTTP-specific + * 6-arg override. + */ + @Override + public IIncomingTransport getFilePullTransport(Node remote, Node local, String securityToken, Map requestProperties, + String registrationUrl, Long resumeBatchId) throws IOException { + return getPullTransport(remote, local, securityToken, requestProperties, registrationUrl); + } + @Override public IOutgoingWithResponseTransport getFilePushTransport(Node remote, Node local, String securityToken, String registrationUrl) throws IOException { @@ -79,6 +89,16 @@ public IIncomingTransport getPullTransport(Node remote, Node local, String secur getDirName(ParameterConstants.NODE_OFFLINE_ERROR_DIR, local)); } + /** + * File-based transport has no concept of a resumed pull; ignore {@code resumeBatchId} rather than inheriting {@link HttpTransportManager}'s HTTP-specific + * 6-arg override. + */ + @Override + public IIncomingTransport getPullTransport(Node remote, Node local, String securityToken, Map requestProperties, + String registrationUrl, Long resumeBatchId) throws IOException { + return getPullTransport(remote, local, securityToken, requestProperties, registrationUrl); + } + @Override public IOutgoingWithResponseTransport getPushTransport(Node remote, Node local, String securityToken, String registrationUrl) throws IOException { diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/http/DefaultHttpResumeCache.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/http/DefaultHttpResumeCache.java new file mode 100644 index 0000000000..bae73b8cdd --- /dev/null +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/http/DefaultHttpResumeCache.java @@ -0,0 +1,89 @@ +/** + * Licensed to JumpMind Inc under one or more contributor + * license agreements. See the NOTICE file distributed + * with this work for additional information regarding + * copyright ownership. JumpMind Inc licenses this file + * to you under the GNU Affero General Public License, version 3.0 (AGPLv3) + * (the "License"); you may not use this file except in compliance + * with the License. + * + * You should have received a copy of the GNU Affero General Public License, + * version 3.0 (AGPLv3) along with this library; if not, see + * . + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jumpmind.symmetric.transport.http; + +import java.util.Objects; + +import org.jumpmind.symmetric.ISymmetricEngine; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Open-source default {@link IHttpResumeCache}: a single-slot holder, since a base engine only ever has one batch in flight at a time per (node, queue) pair + * and only the one batch that was in flight when a connection dropped is ever a resume candidate. A Pro implementation may back this with a bounded, + * multi-entry map instead. + */ +public class DefaultHttpResumeCache implements IHttpResumeCache { + private final Logger log = LoggerFactory.getLogger(getClass()); + private ResumeCacheEntry entry; + + /** + * Matches the {@code (ISymmetricEngine)} constructor shape {@code AppUtils.newInstance} requires so a Pro override can use the engine's parameter service; + * the single-slot default has no need for it. + */ + public DefaultHttpResumeCache(ISymmetricEngine engine) { + } + + @Override + public synchronized void put(String nodeId, long batchId, ResumeCacheEntry newEntry) { + if (entry == null || matches(nodeId, batchId)) { + entry = newEntry; + } else { + log.debug("Resume cache slot busy with node {} batch {}; not registering node {} batch {}", + entry.getNodeId(), entry.getBatchId(), nodeId, batchId); + } + } + + @Override + public synchronized ResumeCacheEntry get(String nodeId, long batchId) { + if (matches(nodeId, batchId)) { + return entry; + } + return null; + } + + @Override + public synchronized ResumeCacheEntry getPendingForNode(String nodeId, String queue) { + if (entry != null && entry.getNodeId().equals(nodeId) && Objects.equals(entry.getQueue(), queue)) { + return entry; + } + return null; + } + + @Override + public synchronized ResumeCacheEntry getPendingFileSyncEntryForNode(String nodeId) { + if (entry != null && entry.getNodeId().equals(nodeId) && entry.isFileSync()) { + return entry; + } + return null; + } + + @Override + public synchronized void remove(String nodeId, long batchId) { + if (matches(nodeId, batchId)) { + entry = null; + } + } + + private boolean matches(String nodeId, long batchId) { + return entry != null && entry.getBatchId() == batchId && entry.getNodeId().equals(nodeId); + } +} diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/http/HttpIncomingTransport.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/http/HttpIncomingTransport.java index 8809c420e1..037f143b3c 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/http/HttpIncomingTransport.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/http/HttpIncomingTransport.java @@ -152,7 +152,7 @@ public InputStream openStream() throws IOException { throw new AuthenticationExpiredException(); case WebConstants.SC_NO_CONTENT: throw new NoContentException(); - case WebConstants.SC_OK: + case WebConstants.SC_OK, WebConstants.SC_PARTIAL_CONTENT: httpTransportManager.updateSession(connection); is = HttpTransportManager.getInputStreamFrom(connection); return is; @@ -168,8 +168,15 @@ public BufferedReader openReader() throws IOException { return reader; } + /** + * {@code URLConnection.getHeaderFields()} doesn't declare {@code throws IOException} - if the underlying connection was never actually established (e.g. + * the network path is down), it silently returns an empty map instead of surfacing the failure. Calling {@code getResponseCode()} first forces the real + * connection attempt through a call that does declare {@code IOException}, so a genuine connectivity failure propagates as an exception rather than being + * indistinguishable from "the server responded without this header". + */ @Override - public Map getHeaders() { + public Map getHeaders() throws IOException { + connection.getResponseCode(); Map headers = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); for (String name : connection.getHeaderFields().keySet()) { if (name != null) { diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/http/HttpTransportManager.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/http/HttpTransportManager.java index 6dc76460bc..d13e14aa13 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/http/HttpTransportManager.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/http/HttpTransportManager.java @@ -53,6 +53,7 @@ import org.jumpmind.symmetric.transport.ITransportManager; import org.jumpmind.symmetric.transport.TransportUtils; import org.jumpmind.symmetric.web.WebConstants; +import org.jumpmind.util.AppUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -67,6 +68,7 @@ public class HttpTransportManager extends AbstractTransportManager implements IT protected boolean useHeaderSecurityToken; protected boolean useSessionAuth; protected int backOffPostCount; + protected IHttpResumeCache resumeCache; public HttpTransportManager() { } @@ -76,6 +78,13 @@ public HttpTransportManager(ISymmetricEngine engine) { this.engine = engine; useHeaderSecurityToken = engine.getParameterService().is(ParameterConstants.TRANSPORT_HTTP_USE_HEADER_SECURITY_TOKEN); useSessionAuth = engine.getParameterService().is(ParameterConstants.TRANSPORT_HTTP_USE_SESSION_AUTH); + resumeCache = AppUtils.newInstance(IHttpResumeCache.class, DefaultHttpResumeCache.class, + new Object[] { engine }, new Class[] { ISymmetricEngine.class }); + } + + @Override + public IHttpResumeCache getResumeCache() { + return resumeCache; } public int sendCopyRequest(Node local) throws IOException { @@ -290,26 +299,49 @@ public void writeMessage(OutputStream out, String data) throws IOException { public IIncomingTransport getFilePullTransport(Node remote, Node local, String securityToken, Map requestProperties, String registrationUrl) throws IOException { - HttpConnection conn = createGetConnectionFor(URI.create(buildURL(WebConstants.URL_FILESYNC_PULL, remote, local, securityToken, registrationUrl)) - .toURL(), - local.getNodeId(), securityToken); + return getFilePullTransport(remote, local, securityToken, requestProperties, registrationUrl, null); + } + + @Override + public IIncomingTransport getFilePullTransport(Node remote, Node local, String securityToken, + Map requestProperties, String registrationUrl, Long resumeBatchId) throws IOException { + String url = buildURL(WebConstants.URL_FILESYNC_PULL, remote, local, securityToken, registrationUrl); + if (resumeBatchId != null) { + url = add(url, WebConstants.BATCH_ID, String.valueOf(resumeBatchId), "&"); + } + HttpConnection conn = createGetConnectionFor(URI.create(url).toURL(), local.getNodeId(), securityToken); if (requestProperties != null) { for (String key : requestProperties.keySet()) { conn.addRequestProperty(key, requestProperties.get(key)); } } + if (log.isDebugEnabled()) { + log.debug("Requesting file pull from {} with headers {}", maskSecurityToken(url), requestProperties); + } return new HttpIncomingTransport(this, conn, engine.getParameterService(), local.getNodeId(), securityToken); } public IIncomingTransport getPullTransport(Node remote, Node local, String securityToken, Map requestProperties, String registrationUrl) throws IOException { - HttpConnection conn = createGetConnectionFor(URI.create(buildURL(WebConstants.URL_PULL, remote, local, securityToken, registrationUrl)).toURL(), - local.getNodeId(), securityToken); + return getPullTransport(remote, local, securityToken, requestProperties, registrationUrl, null); + } + + @Override + public IIncomingTransport getPullTransport(Node remote, Node local, String securityToken, + Map requestProperties, String registrationUrl, Long resumeBatchId) throws IOException { + String url = buildURL(WebConstants.URL_PULL, remote, local, securityToken, registrationUrl); + if (resumeBatchId != null) { + url = add(url, WebConstants.BATCH_ID, String.valueOf(resumeBatchId), "&"); + } + HttpConnection conn = createGetConnectionFor(URI.create(url).toURL(), local.getNodeId(), securityToken); if (requestProperties != null) { for (String key : requestProperties.keySet()) { conn.addRequestProperty(key, requestProperties.get(key)); } } + if (log.isDebugEnabled()) { + log.debug("Requesting pull from {} with headers {}", maskSecurityToken(url), requestProperties); + } return new HttpIncomingTransport(this, conn, engine.getParameterService(), local.getNodeId(), securityToken); } @@ -516,6 +548,13 @@ protected String addNodeInfo(String base, String nodeId, String securityToken, b return sb.toString(); } + /** + * @return {@code url} with any {@code securitytoken} query-string value replaced by {@code ***}, for safe inclusion in debug logging + */ + private static String maskSecurityToken(String url) { + return url.replaceAll("([?&]" + WebConstants.SECURITY_TOKEN + "=)[^&]*", "$1***"); + } + protected String addNodeId(String base, String nodeId, String connector) { return add(base, WebConstants.NODE_ID, nodeId, connector); } diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/http/IHttpResumeCache.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/http/IHttpResumeCache.java new file mode 100644 index 0000000000..4db45f2487 --- /dev/null +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/http/IHttpResumeCache.java @@ -0,0 +1,54 @@ +/** + * Licensed to JumpMind Inc under one or more contributor + * license agreements. See the NOTICE file distributed + * with this work for additional information regarding + * copyright ownership. JumpMind Inc licenses this file + * to you under the GNU Affero General Public License, version 3.0 (AGPLv3) + * (the "License"); you may not use this file except in compliance + * with the License. + * + * You should have received a copy of the GNU Affero General Public License, + * version 3.0 (AGPLv3) along with this library; if not, see + * . + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jumpmind.symmetric.transport.http; + +/** + * Tracks the one (or, in a Pro implementation, several) batch pull(s) currently in flight for a node, so that if the connection drops mid-transfer, the next + * pull attempt can resume from where it left off instead of re-downloading the whole batch. Scoped per-engine, held as an instance field on + * {@link HttpTransportManager}, and resolved via {@link AppUtils#newInstance(Class, Class)} so a Pro engine can substitute a multi-entry implementation with + * zero open-source changes. + */ +public interface IHttpResumeCache { + void put(String nodeId, long batchId, ResumeCacheEntry entry); + + ResumeCacheEntry get(String nodeId, long batchId); + + /** + * @return whichever pending resume entry exists for this node and queue, if any, so a caller can discover a batch worth resuming without already + * knowing its batch id. Scoped by queue as well as node because a single node can have multiple active pull queues (e.g. "default" and "system") - + * without the queue filter, every queue's pull loop would independently discover the same entry and attach its resume parameters to a request that + * has nothing to do with the batch in question. Only meaningful for table-sync, whose channels each belong to exactly one queue and whose batch + * selection is itself queue-partitioned end to end ({@code PullUriHandler} passes the requesting queue into + * {@code DataExtractorService.extract(...)}, and even validates it against the batch's own channel before honoring a resume) - see + * {@link #getPendingFileSyncEntryForNode(String)} for file sync, whose batch selection is not partitioned by queue at all. + */ + ResumeCacheEntry getPendingForNode(String nodeId, String queue); + + /** + * Same intent as {@link #getPendingForNode(String, String)}, but ignores queue entirely and matches only on {@link ResumeCacheEntry#isFileSync()}. Used + * exclusively by file sync: unlike table-sync, {@code FileSyncService.getBatchesToProcess(Node)} selects every outstanding batch across all file-sync + * channels for a node regardless of which queue is asking, so any of that node's active pull queues may legitimately end up delivering - and therefore + * should be able to resume - the one pending file-sync batch, not just the specific queue whose pull happened to be interrupted. + */ + ResumeCacheEntry getPendingFileSyncEntryForNode(String nodeId); + + void remove(String nodeId, long batchId); +} diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/http/ResumeCacheEntry.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/http/ResumeCacheEntry.java new file mode 100644 index 0000000000..950f6e3d67 --- /dev/null +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/transport/http/ResumeCacheEntry.java @@ -0,0 +1,163 @@ +/** + * Licensed to JumpMind Inc under one or more contributor + * license agreements. See the NOTICE file distributed + * with this work for additional information regarding + * copyright ownership. JumpMind Inc licenses this file + * to you under the GNU Affero General Public License, version 3.0 (AGPLv3) + * (the "License"); you may not use this file except in compliance + * with the License. + * + * You should have received a copy of the GNU Affero General Public License, + * version 3.0 (AGPLv3) along with this library; if not, see + * . + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jumpmind.symmetric.transport.http; + +import org.jumpmind.symmetric.io.stage.StagedResourceETag; + +/** + * Identifies one batch pull that was interrupted partway through and is still eligible for a resumed retry. The {@code etag} is captured as soon as it is read + * from the batch's proactive {@code ETAG} preamble line, since the connection can drop before the server gets a chance to send it again. {@code channelId} and + * {@code binaryEncoding} are captured from the same preamble so a resumed ({@code 206}) response - which contains only the remaining row data - can reconstruct + * the batch's identity locally. + *

+ * {@code queue} lets {@link IHttpResumeCache#getPendingForNode(String, String)} restrict a lookup to the one queue that owns this batch, but it's not a + * sufficient discriminator on its own: table-sync and file-sync channels commonly share a queue. {@code fileSync} is the unambiguous marker, set directly by + * the registering code, that each resume consumer uses to recognize only the entries it registered itself. + */ +public class ResumeCacheEntry { + private final String nodeId; + private final long batchId; + private final StagedResourceETag etag; + private final long receivedCount; + private final String channelId; + private final String binaryEncoding; + private final long cachedAtTime; + private final String queue; + private final boolean fileSync; + + private ResumeCacheEntry(Builder builder) { + this.nodeId = builder.nodeId; + this.batchId = builder.batchId; + this.etag = builder.etag; + this.receivedCount = builder.receivedCount; + this.channelId = builder.channelId; + this.binaryEncoding = builder.binaryEncoding; + this.cachedAtTime = builder.cachedAtTime; + this.queue = builder.queue; + this.fileSync = builder.fileSync; + } + + public static Builder builder() { + return new Builder(); + } + + public String getNodeId() { + return nodeId; + } + + public long getBatchId() { + return batchId; + } + + public StagedResourceETag getEtag() { + return etag; + } + + /** + * @return how much of the batch was already received when it was preserved for resume. For a table-sync entry ({@code fileSync == false}) this is a count + * of decoded characters of the staged UTF-8 CSV text, matching what {@code CountingSkippingWriter} skips server-side. For a file-sync entry + * ({@code fileSync == true}) this is a count of raw bytes of the staged zip, matching what {@code IOUtils.skipFully} skips server-side. + */ + public long getReceivedCount() { + return receivedCount; + } + + public String getChannelId() { + return channelId; + } + + public String getBinaryEncoding() { + return binaryEncoding; + } + + public long getCachedAtTime() { + return cachedAtTime; + } + + public String getQueue() { + return queue; + } + + public boolean isFileSync() { + return fileSync; + } + + public static class Builder { + private String nodeId; + private long batchId; + private StagedResourceETag etag; + private long receivedCount; + private String channelId; + private String binaryEncoding; + private long cachedAtTime; + private String queue; + private boolean fileSync; + + public Builder nodeId(String nodeId) { + this.nodeId = nodeId; + return this; + } + + public Builder batchId(long batchId) { + this.batchId = batchId; + return this; + } + + public Builder etag(StagedResourceETag etag) { + this.etag = etag; + return this; + } + + public Builder receivedCount(long receivedCount) { + this.receivedCount = receivedCount; + return this; + } + + public Builder channelId(String channelId) { + this.channelId = channelId; + return this; + } + + public Builder binaryEncoding(String binaryEncoding) { + this.binaryEncoding = binaryEncoding; + return this; + } + + public Builder cachedAtTime(long cachedAtTime) { + this.cachedAtTime = cachedAtTime; + return this; + } + + public Builder queue(String queue) { + this.queue = queue; + return this; + } + + public Builder fileSync(boolean fileSync) { + this.fileSync = fileSync; + return this; + } + + public ResumeCacheEntry build() { + return new ResumeCacheEntry(this); + } + } +} diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/web/WebConstants.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/web/WebConstants.java index 163a27fc19..208fdb8fdd 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/web/WebConstants.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/web/WebConstants.java @@ -56,6 +56,7 @@ public class WebConstants { public static final int SC_INTERNAL_ERROR = 600; public static final int SC_NO_CONTENT = 204; public static final int SC_OK = 200; + public static final int SC_PARTIAL_CONTENT = 206; public static final String ACK_BATCH_NAME = "batch-"; public static final String ACK_BATCH_OK = "ok"; public static final String ACK_BATCH_RESEND = "resend"; @@ -83,6 +84,7 @@ public class WebConstants { public static final String ACK_MISSING_DELETE_COUNT = "missingDeleteCount-"; public static final String ACK_SKIP_COUNT = "skipCount-"; public static final String NODE_ID = "nodeId"; + public static final String BATCH_ID = "batchId"; public static final String NODE_GROUP_ID = "nodeGroupId"; public static final String EXTERNAL_ID = "externalId"; public static final String SYMMETRIC_VERSION = "symmetricVersion"; @@ -104,6 +106,12 @@ public class WebConstants { public static final String HEADER_SECURITY_TOKEN = "Security-Token"; public static final String HEADER_SESSION_ID = "Session-ID"; public static final String HEADER_SET_SESSION_ID = "Set-Session-ID"; + public static final String HEADER_ETAG = "ETag"; + public static final String HEADER_IF_ETAG = "If-ETag"; + public static final String HEADER_ACCEPT_RANGES = "Accept-Ranges"; + public static final String HEADER_RANGE = "Range"; + public static final String HEADER_CONTENT_RANGE = "Content-Range"; + public static final String HEADER_FILESYNC_FORMAT = "FileSync-Format"; public static final String REG_USER_ID = "regUserId"; public static final String REG_PASSWORD = "regPassword"; public static final String PUSH_REGISTRATION = "pushRegistration"; diff --git a/symmetric-core/src/main/resources/symmetric-default.properties b/symmetric-core/src/main/resources/symmetric-default.properties index 34b74de797..41e25fb57b 100644 --- a/symmetric-core/src/main/resources/symmetric-default.properties +++ b/symmetric-core/src/main/resources/symmetric-default.properties @@ -612,6 +612,17 @@ stream.to.file.min.ttl.ms=1800000 # Type: boolean stream.to.file.purge.on.ttl.enabled=false +# Allow a single interrupted batch pull to resume from where it left off using +# HTTP Range/ETag-style headers, instead of re-downloading the whole batch. +# Requires stream.to.file.enabled=true and the batch to be over +# stream.to.file.threshold.bytes; otherwise a full re-download always occurs. +# Older (pre-3.18) peers are unaffected; they simply ignore the extra headers. +# +# DatabaseOverridable: true +# Tags: transport +# Type: boolean +sync.http.resume.enabled=true + # This is the number of times we will attempt to send an ACK back to the remote node # when pulling and loading data. # diff --git a/symmetric-core/src/test/java/org/jumpmind/symmetric/extract/CountingSkippingWriterTest.java b/symmetric-core/src/test/java/org/jumpmind/symmetric/extract/CountingSkippingWriterTest.java new file mode 100644 index 0000000000..d39b191278 --- /dev/null +++ b/symmetric-core/src/test/java/org/jumpmind/symmetric/extract/CountingSkippingWriterTest.java @@ -0,0 +1,103 @@ +/** + * Licensed to JumpMind Inc under one or more contributor + * license agreements. See the NOTICE file distributed + * with this work for additional information regarding + * copyright ownership. JumpMind Inc licenses this file + * to you under the GNU Affero General Public License, version 3.0 (AGPLv3) + * (the "License"); you may not use this file except in compliance + * with the License. + * + * You should have received a copy of the GNU Affero General Public License, + * version 3.0 (AGPLv3) along with this library; if not, see + * . + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jumpmind.symmetric.extract; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.IOException; +import java.io.StringWriter; + +import org.junit.jupiter.api.Test; + +class CountingSkippingWriterTest { + @Test + void testWrite_withZeroSkipCount_forwardsEverything() throws IOException { + StringWriter delegate = new StringWriter(); + try (CountingSkippingWriter writer = new CountingSkippingWriter(delegate, 0)) { + writer.write("hello world".toCharArray(), 0, 11); + assertEquals("hello world", delegate.toString()); + assertEquals(11, writer.getTotalCount()); + } + } + + @Test + void testWrite_withSkipCountWithinFirstBuffer_skipsPartial() throws IOException { + StringWriter delegate = new StringWriter(); + try (CountingSkippingWriter writer = new CountingSkippingWriter(delegate, 6)) { + writer.write("hello world".toCharArray(), 0, 11); + assertEquals("world", delegate.toString()); + assertEquals(11, writer.getTotalCount()); + } + } + + @Test + void testWrite_withSkipCountSpanningMultipleWrites_skipsAcrossBoundary() throws IOException { + StringWriter delegate = new StringWriter(); + try (CountingSkippingWriter writer = new CountingSkippingWriter(delegate, 8)) { + writer.write("hello ".toCharArray(), 0, 6); + writer.write("world".toCharArray(), 0, 5); + assertEquals("rld", delegate.toString()); + assertEquals(11, writer.getTotalCount()); + } + } + + @Test + void testWrite_withSkipCountExactlyMatchingFirstBuffer_forwardsOnlySubsequentWrites() throws IOException { + StringWriter delegate = new StringWriter(); + try (CountingSkippingWriter writer = new CountingSkippingWriter(delegate, 6)) { + writer.write("hello ".toCharArray(), 0, 6); + writer.write("world".toCharArray(), 0, 5); + assertEquals("world", delegate.toString()); + assertEquals(11, writer.getTotalCount()); + } + } + + @Test + void testWrite_withSkipCountGreaterThanTotalContent_forwardsNothingButCountsAll() throws IOException { + StringWriter delegate = new StringWriter(); + try (CountingSkippingWriter writer = new CountingSkippingWriter(delegate, 1000)) { + writer.write("hello world".toCharArray(), 0, 11); + assertEquals("", delegate.toString()); + assertEquals(11, writer.getTotalCount()); + } + } + + @Test + void testWrite_withOffset_honorsOffsetAndLength() throws IOException { + StringWriter delegate = new StringWriter(); + try (CountingSkippingWriter writer = new CountingSkippingWriter(delegate, 2)) { + char[] buffer = "xxhello worldxx".toCharArray(); + writer.write(buffer, 2, 11); + assertEquals("llo world", delegate.toString()); + assertEquals(11, writer.getTotalCount()); + } + } + + @Test + void testFlushAndClose_delegateToUnderlyingWriter() throws IOException { + StringWriter delegate = new StringWriter(); + try (CountingSkippingWriter writer = new CountingSkippingWriter(delegate, 0)) { + writer.write("abc".toCharArray(), 0, 3); + writer.flush(); + } + assertEquals("abc", delegate.toString()); + } +} diff --git a/symmetric-core/src/test/java/org/jumpmind/symmetric/file/FileSyncBatchEnvelopeTest.java b/symmetric-core/src/test/java/org/jumpmind/symmetric/file/FileSyncBatchEnvelopeTest.java new file mode 100644 index 0000000000..a79e73e010 --- /dev/null +++ b/symmetric-core/src/test/java/org/jumpmind/symmetric/file/FileSyncBatchEnvelopeTest.java @@ -0,0 +1,99 @@ +/** + * Licensed to JumpMind Inc under one or more contributor + * license agreements. See the NOTICE file distributed + * with this work for additional information regarding + * copyright ownership. JumpMind Inc licenses this file + * to you under the GNU Affero General Public License, version 3.0 (AGPLv3) + * (the "License"); you may not use this file except in compliance + * with the License. + * + * You should have received a copy of the GNU Affero General Public License, + * version 3.0 (AGPLv3) along with this library; if not, see + * . + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jumpmind.symmetric.file; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import org.jumpmind.symmetric.io.stage.StagedResourceETag; +import org.junit.jupiter.api.Test; + +class FileSyncBatchEnvelopeTest { + @Test + void testWriteThenReadHeaderRoundTrips() throws IOException { + StagedResourceETag etag = new StagedResourceETag(123456789L, 42L); + byte[] body = "some zip bytes, with a comma".getBytes(StandardCharsets.UTF_8); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + FileSyncBatchEnvelope.writeHeader(out, 99L, body.length, etag); + out.write(body); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + FileSyncBatchEnvelope header = FileSyncBatchEnvelope.readHeader(in); + assertEquals(99L, header.getBatchId()); + assertEquals(body.length, header.getLength()); + assertEquals(etag, header.getEtag()); + byte[] readBody = in.readAllBytes(); + assertArrayEquals(body, readBody); + } + + @Test + void testReadHeaderAtCleanEofReturnsNull() throws IOException { + ByteArrayInputStream in = new ByteArrayInputStream(new byte[0]); + assertNull(FileSyncBatchEnvelope.readHeader(in)); + } + + @Test + void testReadHeaderParsesEtagJsonContainingCommas() throws IOException { + StagedResourceETag etag = new StagedResourceETag(1L, 2L); + String etagJson = etag.toJson(); + assertEquals(true, etagJson.contains(",")); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + FileSyncBatchEnvelope.writeHeader(out, 1L, 0L, etag); + FileSyncBatchEnvelope header = FileSyncBatchEnvelope.readHeader(new ByteArrayInputStream(out.toByteArray())); + assertEquals(etag, header.getEtag()); + } + + @Test + void testReadHeaderWithMalformedLineThrowsIOException() { + byte[] malformed = "notavalidheader\n".getBytes(StandardCharsets.UTF_8); + ByteArrayInputStream in = new ByteArrayInputStream(malformed); + assertThrows(IOException.class, () -> FileSyncBatchEnvelope.readHeader(in)); + } + + @Test + void testMultipleHeadersInSequenceReadInOrder() throws IOException { + StagedResourceETag etag1 = new StagedResourceETag(1L, 3L); + StagedResourceETag etag2 = new StagedResourceETag(2L, 4L); + byte[] body1 = "abc".getBytes(StandardCharsets.UTF_8); + byte[] body2 = "wxyz".getBytes(StandardCharsets.UTF_8); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + FileSyncBatchEnvelope.writeHeader(out, 1L, body1.length, etag1); + out.write(body1); + FileSyncBatchEnvelope.writeHeader(out, 2L, body2.length, etag2); + out.write(body2); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + FileSyncBatchEnvelope first = FileSyncBatchEnvelope.readHeader(in); + assertEquals(1L, first.getBatchId()); + byte[] readBody1 = in.readNBytes(body1.length); + assertArrayEquals(body1, readBody1); + FileSyncBatchEnvelope second = FileSyncBatchEnvelope.readHeader(in); + assertEquals(2L, second.getBatchId()); + byte[] readBody2 = in.readNBytes(body2.length); + assertArrayEquals(body2, readBody2); + assertNull(FileSyncBatchEnvelope.readHeader(in)); + } +} diff --git a/symmetric-core/src/test/java/org/jumpmind/symmetric/io/stage/SimpleStagingDataWriterTest.java b/symmetric-core/src/test/java/org/jumpmind/symmetric/io/stage/SimpleStagingDataWriterTest.java new file mode 100644 index 0000000000..98385f2cb6 --- /dev/null +++ b/symmetric-core/src/test/java/org/jumpmind/symmetric/io/stage/SimpleStagingDataWriterTest.java @@ -0,0 +1,479 @@ +/** + * Licensed to JumpMind Inc under one or more contributor + * license agreements. See the NOTICE file distributed + * with this work for additional information regarding + * copyright ownership. JumpMind Inc licenses this file + * to you under the GNU Affero General Public License, version 3.0 (AGPLv3) + * (the "License"); you may not use this file except in compliance + * with the License. + * + * You should have received a copy of the GNU Affero General Public License, + * version 3.0 (AGPLv3) along with this library; if not, see + * . + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jumpmind.symmetric.io.stage; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; +import java.io.Writer; + +import org.apache.commons.io.IOUtils; +import org.jumpmind.db.util.BinaryEncoding; +import org.jumpmind.symmetric.ISymmetricEngine; +import org.jumpmind.symmetric.common.Constants; +import org.jumpmind.symmetric.common.ParameterConstants; +import org.jumpmind.symmetric.io.data.Batch; +import org.jumpmind.symmetric.io.data.Batch.BatchType; +import org.jumpmind.symmetric.io.data.DataContext; +import org.jumpmind.symmetric.io.stage.IStagedResource.State; +import org.jumpmind.symmetric.model.ProcessInfo; +import org.jumpmind.symmetric.model.ProcessInfoKey; +import org.jumpmind.symmetric.model.ProcessType; +import org.jumpmind.symmetric.service.IConfigurationService; +import org.jumpmind.symmetric.service.IParameterService; +import org.jumpmind.symmetric.transport.ITransportManager; +import org.jumpmind.symmetric.transport.http.IHttpResumeCache; +import org.jumpmind.symmetric.transport.http.ResumeCacheEntry; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.ArgumentCaptor; + +class SimpleStagingDataWriterTest { + @TempDir + File tempDir; + private ISymmetricEngine engine; + private IParameterService parameterService; + private ITransportManager transportManager; + private IHttpResumeCache resumeCache; + private StagingManager realStagingManager; + private ProcessInfo processInfo; + private DataContext context; + + @BeforeEach + void setUp() { + engine = mock(ISymmetricEngine.class); + parameterService = mock(IParameterService.class); + when(engine.getParameterService()).thenReturn(parameterService); + transportManager = mock(ITransportManager.class); + when(engine.getTransportManager()).thenReturn(transportManager); + resumeCache = mock(IHttpResumeCache.class); + when(transportManager.getResumeCache()).thenReturn(resumeCache); + IConfigurationService configurationService = mock(IConfigurationService.class); + when(engine.getConfigurationService()).thenReturn(configurationService); + realStagingManager = new StagingManager(tempDir.getAbsolutePath(), false); + when(engine.getStagingManager()).thenReturn(realStagingManager); + processInfo = new ProcessInfo(new ProcessInfoKey("node1", "me", ProcessType.PULL_HANDLER_EXTRACT)); + context = new DataContext(); + context.getContext().put(Constants.DATA_CONTEXT_SOURCE_NODE, "node1"); + } + + private SimpleStagingDataWriter newWriter(String content, ResumeCacheEntry resumeEntry) { + BufferedReader reader = new BufferedReader(new StringReader(content)); + return SimpleStagingDataWriter.builder() + .processInfo(processInfo) + .reader(reader) + .engine(engine) + .category(Constants.STAGING_CATEGORY_INCOMING) + .memoryThresholdInBytes(0L) + .batchType(BatchType.LOAD) + .sourceNodeId("node1") + .targetNodeId("me") + .context(context) + .resumeEntry(resumeEntry) + .build(); + } + + private String readContent(IStagedResource resource) throws IOException { + String content = IOUtils.toString(resource.getReader()); + resource.closeReaders(); + return content; + } + + @Test + void process_happyPath_unaffectedByResumeChanges() throws IOException { + String content = "nodeid,node1\nbinary,NONE\nchannel,channel1\nbatch,100\ninsert,1,foo\ncommit,100\n"; + SimpleStagingDataWriter writer = newWriter(content, null); + writer.process(); + assertNull(writer.getException()); + IStagedResource resource = realStagingManager.find(Constants.STAGING_CATEGORY_INCOMING, "node1", 100L); + assertNotNull(resource); + assertEquals(State.DONE, resource.getState()); + String staged = readContent(resource); + assertTrue(staged.contains("insert,1,foo")); + assertTrue(staged.contains("commit,100")); + verify(resumeCache, never()).put(any(), anyLong(), any()); + verify(resumeCache).remove("node1", 100L); + } + + @Test + void process_etagLineCaptured_butNotPersistedToStagedFile() throws IOException { + String etagJson = new StagedResourceETag(123L, 456L).toJson(); + String content = "nodeid,node1\nbinary,NONE\nchannel,channel1\nbatch,101\netag," + etagJson + "\ninsert,1,foo\ncommit,101\n"; + SimpleStagingDataWriter writer = newWriter(content, null); + writer.process(); + assertNull(writer.getException()); + IStagedResource resource = realStagingManager.find(Constants.STAGING_CATEGORY_INCOMING, "node1", 101L); + String staged = readContent(resource); + assertFalse(staged.contains("etag,")); + assertTrue(staged.contains("insert,1,foo")); + assertTrue(staged.contains("commit,101")); + } + + @Test + void beginResumedBatch_existingCreateStateResource_reopensInAppendModeAndReturnsResource() throws IOException { + Batch preStageBatch = new Batch(BatchType.LOAD, 200L, "channel1", BinaryEncoding.NONE, "node1", "me", false); + IStagedResource existing = realStagingManager.create(Constants.STAGING_CATEGORY_INCOMING, preStageBatch.getStagedLocation(), 200L); + BufferedWriter preWriter = existing.getWriter(0L); + preWriter.write("previously,received\n"); + existing.close(); + StagedResourceETag etag = new StagedResourceETag(111L, 222L); + ResumeCacheEntry resumeEntry = ResumeCacheEntry.builder() + .nodeId("node1") + .batchId(200L) + .etag(etag) + .receivedCount(20L) + .channelId("channel1") + .binaryEncoding("NONE") + .cachedAtTime(999L) + .queue(Constants.QUEUE_DEFAULT) + .build(); + SimpleStagingDataWriter writer = newWriter("insert,2,bar\ncommit,200\n", resumeEntry); + IStagedResource result = writer.beginResumedBatch(); + assertNotNull(result); + assertEquals(200L, writer.batch.getBatchId()); + assertEquals(etag, writer.currentBatchEtag); + assertEquals(0L, writer.stagedCharCount); + assertNotNull(writer.writer); + writer.writer.write("appended,content\n"); + writer.writer.close(); + String staged = readContent(existing); + assertEquals("previously,received\nappended,content\n", staged); + } + + @Test + void beginResumedBatch_missingResource_returnsNullAndLeavesBatchUnset() { + ResumeCacheEntry resumeEntry = ResumeCacheEntry.builder() + .nodeId("node1") + .batchId(201L) + .etag(new StagedResourceETag(1L, 2L)) + .receivedCount(0L) + .channelId("channel1") + .binaryEncoding("NONE") + .cachedAtTime(999L) + .queue(Constants.QUEUE_DEFAULT) + .build(); + SimpleStagingDataWriter writer = newWriter("commit,201\n", resumeEntry); + IStagedResource result = writer.beginResumedBatch(); + assertNull(result); + assertNull(writer.batch); + assertNull(writer.writer); + verify(resumeCache).remove("node1", 201L); + } + + @Test + void beginResumedBatch_resourceAlreadyFinalized_returnsNull() throws IOException { + Batch preStageBatch = new Batch(BatchType.LOAD, 202L, "channel1", BinaryEncoding.NONE, "node1", "me", false); + IStagedResource existing = realStagingManager.create(Constants.STAGING_CATEGORY_INCOMING, preStageBatch.getStagedLocation(), 202L); + BufferedWriter preWriter = existing.getWriter(0L); + preWriter.write("complete\n"); + existing.close(); + existing.setState(State.DONE); + ResumeCacheEntry resumeEntry = ResumeCacheEntry.builder() + .nodeId("node1") + .batchId(202L) + .etag(new StagedResourceETag(1L, 2L)) + .receivedCount(0L) + .channelId("channel1") + .binaryEncoding("NONE") + .cachedAtTime(999L) + .queue(Constants.QUEUE_DEFAULT) + .build(); + SimpleStagingDataWriter writer = newWriter("commit,202\n", resumeEntry); + IStagedResource result = writer.beginResumedBatch(); + assertNull(result); + assertNull(writer.batch); + verify(resumeCache).remove("node1", 202L); + } + + @Test + void process_resumeEntrySupplied_appendsAndCommitsSuccessfully() throws IOException { + Batch preStageBatch = new Batch(BatchType.LOAD, 300L, "channel1", BinaryEncoding.NONE, "node1", "me", false); + IStagedResource existing = realStagingManager.create(Constants.STAGING_CATEGORY_INCOMING, preStageBatch.getStagedLocation(), 300L); + BufferedWriter preWriter = existing.getWriter(0L); + preWriter.write("previously,received\n"); + existing.close(); + ResumeCacheEntry resumeEntry = ResumeCacheEntry.builder() + .nodeId("node1") + .batchId(300L) + .etag(new StagedResourceETag(1L, 2L)) + .receivedCount(20L) + .channelId("channel1") + .binaryEncoding("NONE") + .cachedAtTime(999L) + .queue(Constants.QUEUE_DEFAULT) + .build(); + SimpleStagingDataWriter writer = newWriter("insert,2,bar\ncommit,300\n", resumeEntry); + writer.process(); + assertNull(writer.getException()); + IStagedResource resource = realStagingManager.find(Constants.STAGING_CATEGORY_INCOMING, "node1", 300L); + assertEquals(State.DONE, resource.getState()); + String staged = readContent(resource); + assertEquals("previously,received\ninsert,2,bar\ncommit,300\n", staged); + verify(resumeCache).remove("node1", 300L); + } + + @Test + void isResumableInterruption_allConditionsMet_returnsTrue() { + SimpleStagingDataWriter writer = newWriter("", null); + writer.batch = new Batch(BatchType.LOAD, 1L, "channel1", BinaryEncoding.NONE, "node1", "me", false); + writer.currentBatchEtag = new StagedResourceETag(1L, 2L); + IStagedResource resource = mock(IStagedResource.class); + when(resource.isFileResource()).thenReturn(true); + when(parameterService.is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED)).thenReturn(true); + assertTrue(writer.isResumableInterruption(new IOException("dropped"), resource)); + } + + @Test + void isResumableInterruption_nonIOException_returnsFalse() { + SimpleStagingDataWriter writer = newWriter("", null); + writer.batch = new Batch(BatchType.LOAD, 1L, "channel1", BinaryEncoding.NONE, "node1", "me", false); + writer.currentBatchEtag = new StagedResourceETag(1L, 2L); + IStagedResource resource = mock(IStagedResource.class); + when(resource.isFileResource()).thenReturn(true); + when(parameterService.is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED)).thenReturn(true); + assertFalse(writer.isResumableInterruption(new RuntimeException("data error"), resource)); + } + + @Test + void isResumableInterruption_batchNull_returnsFalse() { + SimpleStagingDataWriter writer = newWriter("", null); + writer.currentBatchEtag = new StagedResourceETag(1L, 2L); + IStagedResource resource = mock(IStagedResource.class); + when(resource.isFileResource()).thenReturn(true); + when(parameterService.is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED)).thenReturn(true); + assertFalse(writer.isResumableInterruption(new IOException("dropped"), resource)); + } + + @Test + void isResumableInterruption_noCurrentEtag_returnsFalse() { + SimpleStagingDataWriter writer = newWriter("", null); + writer.batch = new Batch(BatchType.LOAD, 1L, "channel1", BinaryEncoding.NONE, "node1", "me", false); + IStagedResource resource = mock(IStagedResource.class); + when(resource.isFileResource()).thenReturn(true); + when(parameterService.is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED)).thenReturn(true); + assertFalse(writer.isResumableInterruption(new IOException("dropped"), resource)); + } + + @Test + void isResumableInterruption_resourceNotFileBacked_returnsFalse() { + SimpleStagingDataWriter writer = newWriter("", null); + writer.batch = new Batch(BatchType.LOAD, 1L, "channel1", BinaryEncoding.NONE, "node1", "me", false); + writer.currentBatchEtag = new StagedResourceETag(1L, 2L); + IStagedResource resource = mock(IStagedResource.class); + when(resource.isFileResource()).thenReturn(false); + when(parameterService.is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED)).thenReturn(true); + assertFalse(writer.isResumableInterruption(new IOException("dropped"), resource)); + } + + @Test + void isResumableInterruption_resumeDisabled_returnsFalse() { + SimpleStagingDataWriter writer = newWriter("", null); + writer.batch = new Batch(BatchType.LOAD, 1L, "channel1", BinaryEncoding.NONE, "node1", "me", false); + writer.currentBatchEtag = new StagedResourceETag(1L, 2L); + IStagedResource resource = mock(IStagedResource.class); + when(resource.isFileResource()).thenReturn(true); + when(parameterService.is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED)).thenReturn(false); + assertFalse(writer.isResumableInterruption(new IOException("dropped"), resource)); + } + + @Test + void isResumableInterruption_noResumeCacheAvailable_returnsFalse() { + SimpleStagingDataWriter writer = newWriter("", null); + writer.batch = new Batch(BatchType.LOAD, 1L, "channel1", BinaryEncoding.NONE, "node1", "me", false); + writer.currentBatchEtag = new StagedResourceETag(1L, 2L); + IStagedResource resource = mock(IStagedResource.class); + when(resource.isFileResource()).thenReturn(true); + when(parameterService.is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED)).thenReturn(true); + when(transportManager.getResumeCache()).thenReturn(null); + assertFalse(writer.isResumableInterruption(new IOException("dropped"), resource)); + } + + @Test + void registerForResume_closesResourceAndPutsEntryInCache() { + SimpleStagingDataWriter writer = newWriter("", null); + writer.batch = new Batch(BatchType.LOAD, 55L, "channel1", BinaryEncoding.NONE, "node1", "me", false); + writer.currentBatchEtag = new StagedResourceETag(111L, 222L); + writer.stagedCharCount = 999L; + IStagedResource resource = mock(IStagedResource.class); + writer.registerForResume(resource); + verify(resource).close(); + ArgumentCaptor captor = ArgumentCaptor.forClass(ResumeCacheEntry.class); + verify(resumeCache).put(eq("node1"), eq(55L), captor.capture()); + ResumeCacheEntry entry = captor.getValue(); + assertEquals("node1", entry.getNodeId()); + assertEquals(55L, entry.getBatchId()); + assertEquals(writer.currentBatchEtag, entry.getEtag()); + assertEquals(999L, entry.getReceivedCount()); + assertEquals("channel1", entry.getChannelId()); + assertEquals("NONE", entry.getBinaryEncoding()); + } + + @Test + void registerForResume_withPriorResumeEntry_accumulatesOnTopOfPreviouslyReceivedCount() { + ResumeCacheEntry priorEntry = ResumeCacheEntry.builder() + .nodeId("node1") + .batchId(55L) + .etag(new StagedResourceETag(1L, 2L)) + .receivedCount(500L) + .channelId("channel1") + .binaryEncoding("NONE") + .cachedAtTime(1L) + .queue(Constants.QUEUE_DEFAULT) + .build(); + SimpleStagingDataWriter writer = newWriter("", priorEntry); + writer.batch = new Batch(BatchType.LOAD, 55L, "channel1", BinaryEncoding.NONE, "node1", "me", false); + writer.currentBatchEtag = new StagedResourceETag(111L, 222L); + writer.stagedCharCount = 300L; + IStagedResource resource = mock(IStagedResource.class); + writer.registerForResume(resource); + ArgumentCaptor captor = ArgumentCaptor.forClass(ResumeCacheEntry.class); + verify(resumeCache).put(eq("node1"), eq(55L), captor.capture()); + assertEquals(800L, captor.getValue().getReceivedCount()); + } + + @Test + void writeLine_incrementsStagedCharCountByLineLengthPlusNewline() throws IOException { + SimpleStagingDataWriter writer = newWriter("", null); + writer.writer = new BufferedWriter(new StringWriter()); + writer.writeLine("hello"); + assertEquals(6L, writer.stagedCharCount); + writer.writeLine("world!"); + assertEquals(13L, writer.stagedCharCount); + } + + @Test + void process_bigLineTriggersChunkedWrite_stagedCharCountReflectsFullLength() throws IOException { + String bigValue = "y".repeat(40000); + String content = "nodeid,node1\nbinary,NONE\nchannel,channel1\nbatch,103\ninsert,1," + bigValue + "\ncommit,103\n"; + SimpleStagingDataWriter writer = newWriter(content, null); + writer.process(); + assertNull(writer.getException()); + IStagedResource resource = realStagingManager.find(Constants.STAGING_CATEGORY_INCOMING, "node1", 103L); + String staged = readContent(resource); + String expectedWritten = "nodeid,node1\nbinary,NONE\nchannel,channel1\nbatch,103\ninsert,1," + bigValue + "\ncommit,103\n"; + assertEquals(expectedWritten, staged); + assertEquals(expectedWritten.length(), writer.stagedCharCount); + } + + @Test + void clearResumeCacheEntry_delegatesToResumeCacheRemove() { + SimpleStagingDataWriter writer = newWriter("", null); + writer.clearResumeCacheEntry(42L); + verify(resumeCache).remove("node1", 42L); + } + + @Test + void clearResumeCacheEntry_noTransportManager_doesNotThrow() { + when(engine.getTransportManager()).thenReturn(null); + SimpleStagingDataWriter writer = newWriter("", null); + writer.clearResumeCacheEntry(42L); + verify(resumeCache, never()).remove(anyString(), anyLong()); + } + + /** + * A {@link BufferedWriter} that throws {@link IOException} on a specific, counted {@code write(String)} call, simulating a connection drop partway through + * writing a batch to staging. + */ + private static class DropAfterNWritesWriter extends BufferedWriter { + private int callCount = 0; + private final int failOnCall; + + DropAfterNWritesWriter(Writer out, int failOnCall) { + super(out); + this.failOnCall = failOnCall; + } + + @Override + public void write(String str) throws IOException { + callCount++; + if (callCount == failOnCall) { + throw new IOException("Simulated connection drop"); + } + super.write(str); + } + } + + private SimpleStagingDataWriter newWriterWithPoisonedResource(IStagedResource stagedResource, int failOnWriteCall) { + StagingManager mockStagingManager = mock(StagingManager.class); + when(engine.getStagingManager()).thenReturn(mockStagingManager); + when(mockStagingManager.create(Constants.STAGING_CATEGORY_INCOMING, "node1", 100L)).thenReturn(stagedResource); + when(stagedResource.getWriter(0L)).thenReturn(new DropAfterNWritesWriter(new StringWriter(), failOnWriteCall)); + String etagJson = new StagedResourceETag(123L, 456L).toJson(); + String content = "nodeid,node1\nbinary,NONE\nchannel,channel1\nbatch,100\netag," + etagJson + "\ninsert,1,foo\ncommit,100\n"; + return newWriter(content, null); + } + + @Test + void process_ioExceptionWithResumeEligible_preservesResourceAndRegistersForResume() throws IOException { + IStagedResource stagedResource = mock(IStagedResource.class); + when(stagedResource.isFileResource()).thenReturn(true); + when(parameterService.is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED)).thenReturn(true); + SimpleStagingDataWriter writer = newWriterWithPoisonedResource(stagedResource, 9); + writer.process(); + assertNotNull(writer.getException()); + assertTrue(writer.getException() instanceof IOException); + verify(stagedResource, never()).delete(); + verify(stagedResource).close(); + verify(resumeCache).put(eq("node1"), eq(100L), any(ResumeCacheEntry.class)); + } + + @Test + void process_ioExceptionWithResumeDisabled_deletesResourceAsBefore() throws IOException { + IStagedResource stagedResource = mock(IStagedResource.class); + when(stagedResource.isFileResource()).thenReturn(true); + when(parameterService.is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED)).thenReturn(false); + SimpleStagingDataWriter writer = newWriterWithPoisonedResource(stagedResource, 9); + writer.process(); + assertNotNull(writer.getException()); + verify(stagedResource).delete(); + verify(resumeCache, never()).put(any(), anyLong(), any()); + } + + @Test + void process_ioExceptionButResourceNotFileBacked_deletesResourceAsBefore() throws IOException { + IStagedResource stagedResource = mock(IStagedResource.class); + when(stagedResource.isFileResource()).thenReturn(false); + when(parameterService.is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED)).thenReturn(true); + SimpleStagingDataWriter writer = newWriterWithPoisonedResource(stagedResource, 9); + writer.process(); + assertNotNull(writer.getException()); + verify(stagedResource).delete(); + verify(resumeCache, never()).put(any(), anyLong(), any()); + } +} diff --git a/symmetric-core/src/test/java/org/jumpmind/symmetric/service/impl/DataExtractorServiceResumeRoundTripTest.java b/symmetric-core/src/test/java/org/jumpmind/symmetric/service/impl/DataExtractorServiceResumeRoundTripTest.java new file mode 100644 index 0000000000..488961558b --- /dev/null +++ b/symmetric-core/src/test/java/org/jumpmind/symmetric/service/impl/DataExtractorServiceResumeRoundTripTest.java @@ -0,0 +1,237 @@ +/** + * Licensed to JumpMind Inc under one or more contributor + * license agreements. See the NOTICE file distributed + * with this work for additional information regarding + * copyright ownership. JumpMind Inc licenses this file + * to you under the GNU Affero General Public License, version 3.0 (AGPLv3) + * (the "License"); you may not use this file except in compliance + * with the License. + * + * You should have received a copy of the GNU Affero General Public License, + * version 3.0 (AGPLv3) along with this library; if not, see + * . + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jumpmind.symmetric.service.impl; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; + +import org.apache.commons.io.IOUtils; +import org.jumpmind.db.platform.IDatabasePlatform; +import org.jumpmind.db.sql.ISqlTemplate; +import org.jumpmind.symmetric.ISymmetricEngine; +import org.jumpmind.symmetric.common.Constants; +import org.jumpmind.symmetric.common.ParameterConstants; +import org.jumpmind.symmetric.db.ISymmetricDialect; +import org.jumpmind.symmetric.io.data.Batch; +import org.jumpmind.symmetric.io.data.Batch.BatchType; +import org.jumpmind.symmetric.io.data.CsvConstants; +import org.jumpmind.symmetric.io.data.DataContext; +import org.jumpmind.symmetric.io.stage.IStagedResource; +import org.jumpmind.symmetric.io.stage.IStagedResource.State; +import org.jumpmind.symmetric.io.stage.SimpleStagingDataWriter; +import org.jumpmind.symmetric.io.stage.StagedResourceETag; +import org.jumpmind.symmetric.io.stage.StagingManager; +import org.jumpmind.symmetric.model.Channel; +import org.jumpmind.symmetric.model.Node; +import org.jumpmind.symmetric.model.OutgoingBatch; +import org.jumpmind.symmetric.model.ProcessInfo; +import org.jumpmind.symmetric.model.ProcessInfoKey; +import org.jumpmind.symmetric.model.ProcessType; +import org.jumpmind.symmetric.service.IConfigurationService; +import org.jumpmind.symmetric.service.IDataService; +import org.jumpmind.symmetric.service.INodeService; +import org.jumpmind.symmetric.service.IOutgoingBatchService; +import org.jumpmind.symmetric.service.IParameterService; +import org.jumpmind.symmetric.statistic.IStatisticManager; +import org.jumpmind.symmetric.transport.ITransportManager; +import org.jumpmind.symmetric.transport.http.IHttpResumeCache; +import org.jumpmind.symmetric.transport.http.ResumeCacheEntry; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * End-to-end test of the table-sync resume path: a real {@link DataExtractorService} extracting from a real staged outgoing batch, and a real + * {@link SimpleStagingDataWriter} staging it on the client side. Unlike the granular, per-side tests in {@code DataExtractorServiceTest} and + * {@code SimpleStagingDataWriterTest} (each of which mocks the other side), this exercises both real objects against each other so a coordinate mismatch + * between them can't hide behind a mock. + *

+ * The client's own detection of an interruption and computation of {@code receivedCount} is already covered directly by {@code SimpleStagingDataWriterTest}'s + * {@code registerForResume}/{@code stagedCharCount} tests. This test instead starts from the partial state such an interruption would leave behind (a real, + * partially-written {@code State.CREATE} staged resource, using the same character-counting rule production uses) and exercises the real resume round trip from + * there: the real server-side suppression of the stats/ETag preamble on a resumed extraction, and the real client-side {@code beginResumedBatch()} + * append-and-finalize path, asserting the reassembled staged file is identical to an uninterrupted transfer. + */ +class DataExtractorServiceResumeRoundTripTest { + private static final String MULTI_BYTE_VALUE = "héllo wörld 日本語"; + @TempDir + File serverStagingDir; + @TempDir + File referenceClientStagingDir; + @TempDir + File resumeClientStagingDir; + + @Test + void interruptedResumedTransfer_reassemblesByteIdenticalToUninterruptedTransfer() throws IOException { + IOutgoingBatchService outgoingBatchService = mock(IOutgoingBatchService.class); + StagingManager serverStagingManager = new StagingManager(serverStagingDir.getAbsolutePath(), false); + DataExtractorService dataExtractorService = newServerDataExtractorService(outgoingBatchService, serverStagingManager); + ProcessInfo serverProcessInfo = new ProcessInfo(new ProcessInfoKey("node1", "me", ProcessType.PULL_HANDLER_EXTRACT)); + OutgoingBatch batch = new OutgoingBatch(); + batch.setBatchId(500); + batch.setNodeId("node1"); + batch.setChannelId("channel1"); + String clientStagedPrefix = "nodeid,node1\nbinary,NONE\nchannel,channel1\nbatch,500\n" + + "table,mytable\nkeys,id\ncolumns,id,name\n" + + "insert,1,\"" + MULTI_BYTE_VALUE + "\"\n"; + String stagedContent = clientStagedPrefix + "commit,500\n"; + IStagedResource outResource1 = createOutgoingResource(serverStagingManager, batch, 500L, stagedContent); + IStagedResource outResource2 = createOutgoingResource(serverStagingManager, batch, 600L, stagedContent); + StringWriter fullOut = new StringWriter(); + dataExtractorService.extractSingleBatchForResume(batch, outResource1, fullOut, 0L, serverProcessInfo); + String fullWireContent = fullOut.toString(); + assertEquals(1, batch.getSentCount()); + assertTrue(fullWireContent.contains(CsvConstants.STATS_COLUMNS), "a full (non-resumed) extraction must still include the stats preamble"); + StagingManager referenceClientStagingManager = new StagingManager(referenceClientStagingDir.getAbsolutePath(), false); + SimpleStagingDataWriter referenceWriter = newClientWriter(referenceClientStagingManager, mock(IHttpResumeCache.class), + new BufferedReader(new StringReader(fullWireContent)), null); + referenceWriter.process(); + assertNull(referenceWriter.getException()); + String referenceStaged = readStaged(referenceClientStagingManager, 500L); + assertEquals(stagedContent, referenceStaged); + StagingManager resumeClientStagingManager = new StagingManager(resumeClientStagingDir.getAbsolutePath(), false); + IStagedResource partialResource = resumeClientStagingManager.create(Constants.STAGING_CATEGORY_INCOMING, "node1", 500L); + partialResource.getWriter(0L).write(clientStagedPrefix); + partialResource.close(); + long skipCount = clientStagedPrefix.length(); + StagedResourceETag etag = new StagedResourceETag(outResource1.getGenerationTime(), outResource1.getSize()); + ResumeCacheEntry pendingEntry = ResumeCacheEntry.builder() + .nodeId("node1") + .batchId(500L) + .etag(etag) + .receivedCount(skipCount) + .channelId("channel1") + .binaryEncoding("NONE") + .cachedAtTime(System.currentTimeMillis()) + .queue(Constants.QUEUE_DEFAULT) + .build(); + StringWriter resumedOut = new StringWriter(); + dataExtractorService.extractSingleBatchForResume(batch, outResource2, resumedOut, skipCount, serverProcessInfo); + String resumedWireContent = resumedOut.toString(); + assertEquals(2, batch.getSentCount()); + verify(outgoingBatchService, times(2)).updateOutgoingBatch(batch); + assertEquals("commit,500\n", resumedWireContent, "resumed response must contain exactly the unsent remainder"); + SimpleStagingDataWriter resumedWriter = newClientWriter(resumeClientStagingManager, mock(IHttpResumeCache.class), + new BufferedReader(new StringReader(resumedWireContent)), pendingEntry); + resumedWriter.process(); + assertNull(resumedWriter.getException()); + String resumedStaged = readStaged(resumeClientStagingManager, 500L); + assertEquals(stagedContent, resumedStaged); + assertEquals(referenceStaged, resumedStaged); + } + + private DataExtractorService newServerDataExtractorService(IOutgoingBatchService outgoingBatchService, StagingManager serverStagingManager) { + ISymmetricEngine engine = mock(ISymmetricEngine.class); + when(engine.getTablePrefix()).thenReturn("sym"); + IParameterService parameterService = mock(IParameterService.class); + when(parameterService.getTablePrefix()).thenReturn("sym"); + when(parameterService.is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED)).thenReturn(true); + when(parameterService.getLong(ParameterConstants.OUTGOING_BATCH_UPDATE_STATUS_MILLIS)).thenReturn(Long.MAX_VALUE); + when(engine.getParameterService()).thenReturn(parameterService); + ISymmetricDialect symmetricDialect = mock(ISymmetricDialect.class); + when(symmetricDialect.getName()).thenReturn("H2"); + IDatabasePlatform platform = mock(IDatabasePlatform.class); + ISqlTemplate sqlTemplate = mock(ISqlTemplate.class); + when(platform.getSqlTemplate()).thenReturn(sqlTemplate); + ISqlTemplate sqlTemplateDirty = mock(ISqlTemplate.class); + when(platform.getSqlTemplateDirty()).thenReturn(sqlTemplateDirty); + when(symmetricDialect.getPlatform()).thenReturn(platform); + when(engine.getSymmetricDialect()).thenReturn(symmetricDialect); + when(engine.getDatabasePlatform()).thenReturn(platform); + TriggerRouterService triggerRouterService = mock(TriggerRouterService.class); + when(engine.getTriggerRouterService()).thenReturn(triggerRouterService); + IDataService dataService = mock(IDataService.class); + when(engine.getDataService()).thenReturn(dataService); + INodeService nodeService = mock(INodeService.class); + Node targetNode = new Node(); + targetNode.setNodeId("node1"); + targetNode.setSymmetricVersion("3.18.0"); + when(nodeService.findNode("node1", true)).thenReturn(targetNode); + when(engine.getNodeService()).thenReturn(nodeService); + IConfigurationService configurationService = mock(IConfigurationService.class); + Channel channel = new Channel(); + channel.setChannelId("channel1"); + when(configurationService.getChannel("channel1")).thenReturn(channel); + when(engine.getConfigurationService()).thenReturn(configurationService); + when(engine.getOutgoingBatchService()).thenReturn(outgoingBatchService); + IStatisticManager statisticManager = mock(IStatisticManager.class); + when(engine.getStatisticManager()).thenReturn(statisticManager); + when(engine.getStagingManager()).thenReturn(serverStagingManager); + return new DataExtractorService(engine); + } + + private SimpleStagingDataWriter newClientWriter(StagingManager stagingManager, IHttpResumeCache resumeCache, BufferedReader reader, + ResumeCacheEntry resumeEntry) { + ISymmetricEngine clientEngine = mock(ISymmetricEngine.class); + when(clientEngine.getStagingManager()).thenReturn(stagingManager); + ITransportManager transportManager = mock(ITransportManager.class); + when(transportManager.getResumeCache()).thenReturn(resumeCache); + when(clientEngine.getTransportManager()).thenReturn(transportManager); + IParameterService parameterService = mock(IParameterService.class); + when(parameterService.is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED)).thenReturn(true); + when(clientEngine.getParameterService()).thenReturn(parameterService); + IConfigurationService configurationService = mock(IConfigurationService.class); + when(clientEngine.getConfigurationService()).thenReturn(configurationService); + DataContext context = new DataContext(); + context.getContext().put(Constants.DATA_CONTEXT_SOURCE_NODE, "node1"); + return SimpleStagingDataWriter.builder() + .processInfo(new ProcessInfo(new ProcessInfoKey("node1", "me", ProcessType.PULL_HANDLER_EXTRACT))) + .reader(reader) + .engine(clientEngine) + .category(Constants.STAGING_CATEGORY_INCOMING) + .memoryThresholdInBytes(0L) + .batchType(BatchType.LOAD) + .sourceNodeId("node1") + .targetNodeId("me") + .context(context) + .resumeEntry(resumeEntry) + .build(); + } + + private IStagedResource createOutgoingResource(StagingManager stagingManager, OutgoingBatch batch, long stagingBatchId, String content) + throws IOException { + Batch outgoingBatchDescriptor = new Batch(BatchType.EXTRACT, stagingBatchId, batch.getChannelId(), null, batch.getNodeId(), "me", false); + IStagedResource resource = stagingManager.create(Constants.STAGING_CATEGORY_OUTGOING, outgoingBatchDescriptor.getStagedLocation(), stagingBatchId); + resource.getWriter(0L).write(content); + resource.close(); + resource.setState(State.DONE); + return resource; + } + + private String readStaged(StagingManager stagingManager, long batchId) throws IOException { + IStagedResource resource = stagingManager.find(Constants.STAGING_CATEGORY_INCOMING, "node1", batchId); + assertNotNull(resource); + String content = IOUtils.toString(resource.getReader()); + resource.closeReaders(); + return content; + } +} diff --git a/symmetric-core/src/test/java/org/jumpmind/symmetric/service/impl/DataExtractorServiceTest.java b/symmetric-core/src/test/java/org/jumpmind/symmetric/service/impl/DataExtractorServiceTest.java index 2d0f1befb4..303c6f23b9 100644 --- a/symmetric-core/src/test/java/org/jumpmind/symmetric/service/impl/DataExtractorServiceTest.java +++ b/symmetric-core/src/test/java/org/jumpmind/symmetric/service/impl/DataExtractorServiceTest.java @@ -21,6 +21,9 @@ package org.jumpmind.symmetric.service.impl; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; @@ -36,6 +39,9 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.StringWriter; import java.util.ArrayList; import java.util.Collections; import java.util.Date; @@ -53,12 +59,17 @@ import org.jumpmind.db.sql.ISqlTemplate; import org.jumpmind.db.sql.ISqlTransaction; import org.jumpmind.symmetric.ISymmetricEngine; +import org.jumpmind.symmetric.common.Constants; import org.jumpmind.symmetric.common.ParameterConstants; import org.jumpmind.symmetric.db.ISymmetricDialect; import org.jumpmind.symmetric.extract.SelectFromSymDataSource; +import org.jumpmind.symmetric.io.data.CsvConstants; import org.jumpmind.symmetric.io.data.DataEventType; import org.jumpmind.symmetric.io.data.IDataWriter; +import org.jumpmind.symmetric.io.stage.IStagedResource; +import org.jumpmind.symmetric.io.stage.IStagedResource.State; import org.jumpmind.symmetric.io.stage.IStagingManager; +import org.jumpmind.symmetric.io.stage.StagedResourceETag; import org.jumpmind.symmetric.model.AbstractBatch.Status; import org.jumpmind.symmetric.model.Data; import org.jumpmind.symmetric.model.ExtractRequest; @@ -90,11 +101,13 @@ class DataExtractorServiceTest { private static final long LOAD_ID = 7929; protected ISymmetricEngine engine; - private IParameterService parameterService; + protected IStagingManager stagingManager; + protected INodeService nodeService; + protected IParameterService parameterService; + protected DataExtractorService dataExtractorService; private ISqlTemplate sqlTemplate; private ISqlTemplate sqlTemplateDirty; private IDataService dataService; - private INodeService nodeService; private TestableDataExtractorService service; private Node targetNode; @@ -137,8 +150,11 @@ void setUp() { when(platform.getSqlTemplateDirty()).thenReturn(sqlTemplateDirty); dataService = mock(IDataService.class); when(engine.getDataService()).thenReturn(dataService); + stagingManager = mock(IStagingManager.class); + when(engine.getStagingManager()).thenReturn(stagingManager); nodeService = mock(INodeService.class); when(engine.getNodeService()).thenReturn(nodeService); + dataExtractorService = new DataExtractorService(engine); service = new TestableDataExtractorService(engine); targetNode = new Node(); when(parameterService.is(ParameterConstants.INITIAL_LOAD_DEFER_CREATE_CONSTRAINTS, false)).thenReturn(true); @@ -161,6 +177,158 @@ void selectFromSymDataSource_csvValuesAreExtracted_triggerRouterIsNotMarkedAsMis assertTrue(source.next().equals(data)); } + @Test + void getStagedResourceForResume_nullBatch_returnsNull() { + assertNull(dataExtractorService.getStagedResourceForResume(null)); + } + + @Test + void getStagedResourceForResume_delegatesToStagingManagerLookup() { + OutgoingBatch batch = new OutgoingBatch(); + batch.setBatchId(123); + batch.setNodeId("node1"); + IStagedResource resource = mock(IStagedResource.class); + when(stagingManager.find(Constants.STAGING_CATEGORY_OUTGOING, batch.getStagedLocation(), batch.getBatchId())).thenReturn(resource); + assertEquals(resource, dataExtractorService.getStagedResourceForResume(batch)); + } + + @Test + void getResumeEtagIfEligible_resumeDisabled_returnsNull() { + when(parameterService.is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED)).thenReturn(false); + IStagedResource resource = mock(IStagedResource.class); + assertNull(dataExtractorService.getResumeEtagIfEligible(new OutgoingBatch(), resource)); + } + + @Test + void getResumeEtagIfEligible_nullStagedResource_returnsNull() { + when(parameterService.is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED)).thenReturn(true); + assertNull(dataExtractorService.getResumeEtagIfEligible(new OutgoingBatch(), null)); + } + + @Test + void getResumeEtagIfEligible_resourceNotDone_returnsNull() { + when(parameterService.is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED)).thenReturn(true); + IStagedResource resource = mock(IStagedResource.class); + when(resource.getState()).thenReturn(State.CREATE); + assertNull(dataExtractorService.getResumeEtagIfEligible(new OutgoingBatch(), resource)); + } + + @Test + void getResumeEtagIfEligible_resourceNotFileBacked_returnsNull() { + when(parameterService.is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED)).thenReturn(true); + IStagedResource resource = mock(IStagedResource.class); + when(resource.getState()).thenReturn(State.DONE); + when(resource.isFileResource()).thenReturn(false); + assertNull(dataExtractorService.getResumeEtagIfEligible(new OutgoingBatch(), resource)); + } + + @Test + void getResumeEtagIfEligible_nodeNotFound_returnsNull() { + when(parameterService.is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED)).thenReturn(true); + IStagedResource resource = mock(IStagedResource.class); + when(resource.getState()).thenReturn(State.DONE); + when(resource.isFileResource()).thenReturn(true); + OutgoingBatch batch = new OutgoingBatch(); + batch.setNodeId("node1"); + when(nodeService.findNode("node1", true)).thenReturn(null); + assertNull(dataExtractorService.getResumeEtagIfEligible(batch, resource)); + } + + @Test + void getResumeEtagIfEligible_nodeVersionTooOld_returnsNull() { + when(parameterService.is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED)).thenReturn(true); + IStagedResource resource = mock(IStagedResource.class); + when(resource.getState()).thenReturn(State.DONE); + when(resource.isFileResource()).thenReturn(true); + OutgoingBatch batch = new OutgoingBatch(); + batch.setNodeId("node1"); + Node node = new Node(); + node.setSymmetricVersion("3.17.0"); + when(nodeService.findNode("node1", true)).thenReturn(node); + assertNull(dataExtractorService.getResumeEtagIfEligible(batch, resource)); + } + + @Test + void getResumeEtagIfEligible_allConditionsMet_returnsEtag() { + when(parameterService.is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED)).thenReturn(true); + IStagedResource resource = mock(IStagedResource.class); + when(resource.getState()).thenReturn(State.DONE); + when(resource.isFileResource()).thenReturn(true); + when(resource.getGenerationTime()).thenReturn(1000L); + when(resource.getSize()).thenReturn(2000L); + OutgoingBatch batch = new OutgoingBatch(); + batch.setNodeId("node1"); + Node node = new Node(); + node.setSymmetricVersion("3.18.0"); + when(nodeService.findNode("node1", true)).thenReturn(node); + StagedResourceETag etag = dataExtractorService.getResumeEtagIfEligible(batch, resource); + assertNotNull(etag); + assertEquals(1000L, etag.getGenerationTime()); + assertEquals(2000L, etag.getSize()); + } + + @Test + void writeBatchPreambleExtras_neitherStatsNorEtag_writesBufferUnchanged() throws IOException { + String content = "\n" + CsvConstants.BATCH + ",1\ndata after batch line"; + char[] buffer = content.toCharArray(); + StringWriter stringWriter = new StringWriter(); + try (BufferedWriter writer = new BufferedWriter(stringWriter)) { + boolean injected = dataExtractorService.writeBatchPreambleExtras(writer, buffer, buffer.length, "", new OutgoingBatch(), false, null); + writer.flush(); + assertTrue(injected); + assertEquals(content, stringWriter.toString()); + } + } + + @Test + void writeBatchPreambleExtras_etagOnly_injectsEtagLineAfterBatchLine() throws IOException { + String content = "\n" + CsvConstants.BATCH + ",1\ndata after batch line"; + char[] buffer = content.toCharArray(); + StringWriter stringWriter = new StringWriter(); + StagedResourceETag etag = new StagedResourceETag(1000L, 2000L); + try (BufferedWriter writer = new BufferedWriter(stringWriter)) { + boolean injected = dataExtractorService.writeBatchPreambleExtras(writer, buffer, buffer.length, "", new OutgoingBatch(), false, etag); + writer.flush(); + assertTrue(injected); + String result = stringWriter.toString(); + assertTrue(result.contains(CsvConstants.ETAG + "," + etag.toJson())); + assertTrue(result.endsWith("data after batch line")); + } + } + + @Test + void writeBatchPreambleExtras_statsAndEtag_injectsBothInOrder() throws IOException { + String content = "\n" + CsvConstants.BATCH + ",1\ndata after batch line"; + char[] buffer = content.toCharArray(); + StringWriter stringWriter = new StringWriter(); + StagedResourceETag etag = new StagedResourceETag(1000L, 2000L); + OutgoingBatch batch = new OutgoingBatch(); + try (BufferedWriter writer = new BufferedWriter(stringWriter)) { + boolean injected = dataExtractorService.writeBatchPreambleExtras(writer, buffer, buffer.length, "", batch, true, etag); + writer.flush(); + assertTrue(injected); + String result = stringWriter.toString(); + int statsIndex = result.indexOf(CsvConstants.STATS_COLUMNS); + int etagIndex = result.indexOf(CsvConstants.ETAG + ","); + assertTrue(statsIndex >= 0); + assertTrue(etagIndex > statsIndex); + } + } + + @Test + void writeBatchPreambleExtras_noBatchLineFound_writesBufferUnchangedAndReturnsFalse() throws IOException { + String content = "no batch marker in this text"; + char[] buffer = content.toCharArray(); + StringWriter stringWriter = new StringWriter(); + try (BufferedWriter writer = new BufferedWriter(stringWriter)) { + boolean injected = dataExtractorService.writeBatchPreambleExtras(writer, buffer, buffer.length, "", new OutgoingBatch(), true, + new StagedResourceETag(1L, 2L)); + writer.flush(); + assertFalse(injected); + assertEquals(content, stringWriter.toString()); + } + } + @Test void checkSendDeferredForeignKeys_deferConstraintsDisabled_neverSends() { when(parameterService.is(ParameterConstants.INITIAL_LOAD_DEFER_CREATE_CONSTRAINTS, false)).thenReturn(false); diff --git a/symmetric-core/src/test/java/org/jumpmind/symmetric/service/impl/DataLoaderServiceTest.java b/symmetric-core/src/test/java/org/jumpmind/symmetric/service/impl/DataLoaderServiceTest.java new file mode 100644 index 0000000000..de4cd78316 --- /dev/null +++ b/symmetric-core/src/test/java/org/jumpmind/symmetric/service/impl/DataLoaderServiceTest.java @@ -0,0 +1,254 @@ +/** + * Licensed to JumpMind Inc under one or more contributor + * license agreements. See the NOTICE file distributed + * with this work for additional information regarding + * copyright ownership. JumpMind Inc licenses this file + * to you under the GNU Affero General Public License, version 3.0 (AGPLv3) + * (the "License"); you may not use this file except in compliance + * with the License. + * + * You should have received a copy of the GNU Affero General Public License, + * version 3.0 (AGPLv3) along with this library; if not, see + * . + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jumpmind.symmetric.service.impl; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import org.jumpmind.db.platform.IDatabasePlatform; +import org.jumpmind.db.sql.ISqlTemplate; +import org.jumpmind.symmetric.ISymmetricEngine; +import org.jumpmind.symmetric.cache.ICacheManager; +import org.jumpmind.symmetric.common.Constants; +import org.jumpmind.symmetric.db.ISymmetricDialect; +import org.jumpmind.symmetric.io.stage.StagedResourceETag; +import org.jumpmind.symmetric.model.Node; +import org.jumpmind.symmetric.model.NodeChannels; +import org.jumpmind.symmetric.model.NodeSecurity; +import org.jumpmind.symmetric.model.ProcessInfo; +import org.jumpmind.symmetric.model.ProcessInfoKey; +import org.jumpmind.symmetric.model.ProcessType; +import org.jumpmind.symmetric.model.RemoteNodeStatus; +import org.jumpmind.symmetric.service.IConfigurationService; +import org.jumpmind.symmetric.service.IExtensionService; +import org.jumpmind.symmetric.service.IIncomingBatchService; +import org.jumpmind.symmetric.service.ILoadFilterService; +import org.jumpmind.symmetric.service.INodeCommunicationService; +import org.jumpmind.symmetric.service.INodeService; +import org.jumpmind.symmetric.service.IParameterService; +import org.jumpmind.symmetric.service.ITransformService; +import org.jumpmind.symmetric.statistic.IStatisticManager; +import org.jumpmind.symmetric.transport.IIncomingTransport; +import org.jumpmind.symmetric.transport.ITransportManager; +import org.jumpmind.symmetric.transport.http.IHttpResumeCache; +import org.jumpmind.symmetric.transport.http.ResumeCacheEntry; +import org.jumpmind.symmetric.web.WebConstants; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +class DataLoaderServiceTest { + private ISymmetricEngine engine; + private IParameterService parameterService; + private INodeService nodeService; + private IConfigurationService configurationService; + private ITransportManager transportManager; + private IHttpResumeCache resumeCache; + private DataLoaderService dataLoaderService; + private Node remote; + private Node local; + private RemoteNodeStatus status; + private IIncomingTransport transport; + + @BeforeEach + void setUp() throws Exception { + engine = mock(ISymmetricEngine.class); + when(engine.getTablePrefix()).thenReturn("sym"); + parameterService = mock(IParameterService.class); + when(parameterService.getTablePrefix()).thenReturn("sym"); + when(parameterService.getRegistrationUrl()).thenReturn("http://registration"); + when(engine.getParameterService()).thenReturn(parameterService); + ISymmetricDialect symmetricDialect = mock(ISymmetricDialect.class); + IDatabasePlatform platform = mock(IDatabasePlatform.class); + ISqlTemplate sqlTemplate = mock(ISqlTemplate.class); + when(platform.getSqlTemplate()).thenReturn(sqlTemplate); + ISqlTemplate sqlTemplateDirty = mock(ISqlTemplate.class); + when(platform.getSqlTemplateDirty()).thenReturn(sqlTemplateDirty); + when(symmetricDialect.getPlatform()).thenReturn(platform); + when(engine.getSymmetricDialect()).thenReturn(symmetricDialect); + IIncomingBatchService incomingBatchService = mock(IIncomingBatchService.class); + when(engine.getIncomingBatchService()).thenReturn(incomingBatchService); + configurationService = mock(IConfigurationService.class); + when(engine.getConfigurationService()).thenReturn(configurationService); + transportManager = mock(ITransportManager.class); + when(engine.getTransportManager()).thenReturn(transportManager); + resumeCache = mock(IHttpResumeCache.class); + when(transportManager.getResumeCache()).thenReturn(resumeCache); + IStatisticManager statisticManager = mock(IStatisticManager.class); + when(statisticManager.newProcessInfo(any())).thenReturn(new ProcessInfo( + new ProcessInfoKey("remote1", Constants.QUEUE_DEFAULT, "me", ProcessType.PULL_JOB_TRANSFER))); + when(engine.getStatisticManager()).thenReturn(statisticManager); + nodeService = mock(INodeService.class); + when(engine.getNodeService()).thenReturn(nodeService); + ITransformService transformService = mock(ITransformService.class); + when(engine.getTransformService()).thenReturn(transformService); + ILoadFilterService loadFilterService = mock(ILoadFilterService.class); + when(engine.getLoadFilterService()).thenReturn(loadFilterService); + IExtensionService extensionService = mock(IExtensionService.class); + when(engine.getExtensionService()).thenReturn(extensionService); + INodeCommunicationService nodeCommunicationService = mock(INodeCommunicationService.class); + when(engine.getNodeCommunicationService()).thenReturn(nodeCommunicationService); + ICacheManager cacheManager = mock(ICacheManager.class); + when(engine.getCacheManager()).thenReturn(cacheManager); + dataLoaderService = spy(new DataLoaderService(engine)); + doReturn(Collections.emptyList()).when(dataLoaderService).loadDataFromTransport(any(), any(), any(), any(), any(), any()); + remote = new Node(); + remote.setNodeId("remote1"); + local = new Node(); + local.setNodeId("me"); + when(nodeService.findIdentity()).thenReturn(local); + NodeSecurity localSecurity = new NodeSecurity(); + when(nodeService.findNodeSecurity("me", true)).thenReturn(localSecurity); + when(configurationService.getSuspendIgnoreChannelLists("remote1")).thenReturn(new NodeChannels()); + status = new RemoteNodeStatus("remote1", Constants.CHANNEL_DEFAULT, new HashMap<>()); + transport = mock(IIncomingTransport.class); + when(transportManager.getPullTransport(any(), any(), any(), anyMap(), any(), any())).thenReturn(transport); + } + + @Test + void loadDataFromPull_noPendingResume_pullsNormallyWithNullResumeBatchId() throws Exception { + when(resumeCache.getPendingForNode("remote1", Constants.QUEUE_DEFAULT)).thenReturn(null); + when(transport.getHeaders()).thenReturn(new HashMap<>()); + dataLoaderService.loadDataFromPull(remote, status); + @SuppressWarnings("unchecked") + ArgumentCaptor> propsCaptor = ArgumentCaptor.forClass(Map.class); + verify(transportManager).getPullTransport(eq(remote), eq(local), any(), propsCaptor.capture(), any(), isNull()); + assertNull(propsCaptor.getValue().get(WebConstants.HEADER_IF_ETAG)); + assertNull(propsCaptor.getValue().get(WebConstants.HEADER_RANGE)); + verify(dataLoaderService).loadDataFromTransport(any(), eq(remote), eq(transport), isNull(), eq(status), isNull()); + verify(resumeCache, never()).remove(any(), any(Long.class)); + } + + @Test + void loadDataFromPull_pendingResume_addsIfETagAndRangeHeadersAndRequestsResumeBatchId() throws Exception { + StagedResourceETag etag = new StagedResourceETag(111L, 500L); + ResumeCacheEntry pendingResume = ResumeCacheEntry.builder() + .nodeId("remote1") + .batchId(77L) + .etag(etag) + .receivedCount(200L) + .channelId("channel1") + .binaryEncoding("NONE") + .cachedAtTime(123L) + .queue(Constants.QUEUE_DEFAULT) + .build(); + when(resumeCache.getPendingForNode("remote1", Constants.QUEUE_DEFAULT)).thenReturn(pendingResume); + Map responseHeaders = new HashMap<>(); + responseHeaders.put(WebConstants.HEADER_CONTENT_RANGE, "200-499/500"); + when(transport.getHeaders()).thenReturn(responseHeaders); + dataLoaderService.loadDataFromPull(remote, status); + @SuppressWarnings("unchecked") + ArgumentCaptor> propsCaptor = ArgumentCaptor.forClass(Map.class); + verify(transportManager).getPullTransport(eq(remote), eq(local), any(), propsCaptor.capture(), any(), eq(77L)); + assertEquals(etag.toJson(), propsCaptor.getValue().get(WebConstants.HEADER_IF_ETAG)); + assertEquals("chars=200-", propsCaptor.getValue().get(WebConstants.HEADER_RANGE)); + verify(dataLoaderService).loadDataFromTransport(any(), eq(remote), eq(transport), isNull(), eq(status), eq(pendingResume)); + verify(resumeCache, never()).remove(any(), any(Long.class)); + } + + @Test + void loadDataFromPull_serverDeclinesResume_clearsCacheAndFallsBackToNormalLoad() throws Exception { + StagedResourceETag etag = new StagedResourceETag(111L, 500L); + ResumeCacheEntry pendingResume = ResumeCacheEntry.builder() + .nodeId("remote1") + .batchId(77L) + .etag(etag) + .receivedCount(200L) + .channelId("channel1") + .binaryEncoding("NONE") + .cachedAtTime(123L) + .queue(Constants.QUEUE_DEFAULT) + .build(); + when(resumeCache.getPendingForNode("remote1", Constants.QUEUE_DEFAULT)).thenReturn(pendingResume); + when(transport.getHeaders()).thenReturn(new HashMap<>()); + dataLoaderService.loadDataFromPull(remote, status); + verify(resumeCache).remove("remote1", 77L); + verify(dataLoaderService).loadDataFromTransport(any(), eq(remote), eq(transport), isNull(), eq(status), isNull()); + } + + @Test + void loadDataFromPull_noResumeCacheAvailable_pullsNormallyWithoutNpe() throws Exception { + when(transportManager.getResumeCache()).thenReturn(null); + when(transport.getHeaders()).thenReturn(new HashMap<>()); + dataLoaderService.loadDataFromPull(remote, status); + verify(transportManager).getPullTransport(eq(remote), eq(local), any(), anyMap(), any(), isNull()); + verify(dataLoaderService).loadDataFromTransport(any(), eq(remote), eq(transport), isNull(), eq(status), isNull()); + } + + @Test + void loadDataFromPull_pendingResumeBelongsToFileSync_ignoresItAndPullsNormally() throws Exception { + StagedResourceETag etag = new StagedResourceETag(111L, 500L); + ResumeCacheEntry pendingResume = ResumeCacheEntry.builder() + .nodeId("remote1") + .batchId(77L) + .etag(etag) + .receivedCount(200L) + .cachedAtTime(123L) + .queue(Constants.QUEUE_DEFAULT) + .fileSync(true) + .build(); + when(resumeCache.getPendingForNode("remote1", Constants.QUEUE_DEFAULT)).thenReturn(pendingResume); + when(transport.getHeaders()).thenReturn(new HashMap<>()); + dataLoaderService.loadDataFromPull(remote, status); + @SuppressWarnings("unchecked") + ArgumentCaptor> propsCaptor = ArgumentCaptor.forClass(Map.class); + verify(transportManager).getPullTransport(eq(remote), eq(local), any(), propsCaptor.capture(), any(), isNull()); + assertNull(propsCaptor.getValue().get(WebConstants.HEADER_IF_ETAG)); + assertNull(propsCaptor.getValue().get(WebConstants.HEADER_RANGE)); + verify(dataLoaderService).loadDataFromTransport(any(), eq(remote), eq(transport), isNull(), eq(status), isNull()); + verify(resumeCache, never()).remove(any(), any(Long.class)); + } + + @Test + void loadDataFromPull_connectionFailsDuringResumeCheck_preservesCacheEntryAndPropagatesException() throws Exception { + StagedResourceETag etag = new StagedResourceETag(111L, 500L); + ResumeCacheEntry pendingResume = ResumeCacheEntry.builder() + .nodeId("remote1") + .batchId(77L) + .etag(etag) + .receivedCount(200L) + .channelId("channel1") + .binaryEncoding("NONE") + .cachedAtTime(123L) + .queue(Constants.QUEUE_DEFAULT) + .build(); + when(resumeCache.getPendingForNode("remote1", Constants.QUEUE_DEFAULT)).thenReturn(pendingResume); + when(transport.getHeaders()).thenThrow(new IOException("Connection refused")); + assertThrows(IOException.class, () -> dataLoaderService.loadDataFromPull(remote, status)); + verify(resumeCache, never()).remove(any(), any(Long.class)); + } +} diff --git a/symmetric-core/src/test/java/org/jumpmind/symmetric/service/impl/FileSyncServiceTest.java b/symmetric-core/src/test/java/org/jumpmind/symmetric/service/impl/FileSyncServiceTest.java index 951248c0d8..db73aa329d 100644 --- a/symmetric-core/src/test/java/org/jumpmind/symmetric/service/impl/FileSyncServiceTest.java +++ b/symmetric-core/src/test/java/org/jumpmind/symmetric/service/impl/FileSyncServiceTest.java @@ -20,8 +20,17 @@ */ package org.jumpmind.symmetric.service.impl; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -31,30 +40,524 @@ import static org.mockito.Mockito.when; import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.zip.ZipOutputStream; import org.jumpmind.db.platform.IDatabasePlatform; +import org.jumpmind.db.sql.ISqlTemplate; import org.jumpmind.symmetric.ISymmetricEngine; import org.jumpmind.symmetric.cache.ICacheManager; +import org.jumpmind.symmetric.common.Constants; +import org.jumpmind.symmetric.common.ParameterConstants; import org.jumpmind.symmetric.db.ISymmetricDialect; +import org.jumpmind.symmetric.file.FileSyncBatchEnvelope; +import org.jumpmind.symmetric.file.FileSyncPullResult; import org.jumpmind.symmetric.io.stage.IStagedResource; +import org.jumpmind.symmetric.io.stage.IStagedResource.State; +import org.jumpmind.symmetric.io.stage.IStagingManager; +import org.jumpmind.symmetric.io.stage.StagedResourceETag; import org.jumpmind.symmetric.model.AbstractBatch.Status; +import org.jumpmind.symmetric.model.Channel; +import org.jumpmind.symmetric.model.IncomingBatch; import org.jumpmind.symmetric.model.Node; +import org.jumpmind.symmetric.model.NodeCommunication; +import org.jumpmind.symmetric.model.NodeSecurity; import org.jumpmind.symmetric.model.OutgoingBatch; import org.jumpmind.symmetric.model.ProcessInfo; import org.jumpmind.symmetric.model.ProcessInfoKey; +import org.jumpmind.symmetric.model.RemoteNodeStatus; import org.jumpmind.symmetric.service.IConfigurationService; import org.jumpmind.symmetric.service.IExtensionService; +import org.jumpmind.symmetric.service.INodeService; import org.jumpmind.symmetric.service.IOutgoingBatchService; import org.jumpmind.symmetric.service.IParameterService; import org.jumpmind.symmetric.statistic.IStatisticManager; +import org.jumpmind.symmetric.transport.IIncomingTransport; import org.jumpmind.symmetric.transport.IOutgoingTransport; +import org.jumpmind.symmetric.transport.ITransportManager; +import org.jumpmind.symmetric.transport.http.IHttpResumeCache; +import org.jumpmind.symmetric.transport.http.ResumeCacheEntry; +import org.jumpmind.symmetric.web.WebConstants; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; class FileSyncServiceTest { + private ISymmetricEngine engine; + private IParameterService parameterService; + private IStagingManager stagingManager; + private IOutgoingBatchService outgoingBatchService; + private IHttpResumeCache resumeCache; + private FileSyncService fileSyncService; + + @BeforeEach + void setUp() { + engine = mock(ISymmetricEngine.class); + parameterService = mock(IParameterService.class); + when(parameterService.getTablePrefix()).thenReturn("sym"); + when(parameterService.getTempDirectory()).thenReturn(System.getProperty("java.io.tmpdir")); + when(engine.getParameterService()).thenReturn(parameterService); + ISymmetricDialect symmetricDialect = mock(ISymmetricDialect.class); + IDatabasePlatform platform = mock(IDatabasePlatform.class); + ISqlTemplate sqlTemplate = mock(ISqlTemplate.class); + when(platform.getSqlTemplate()).thenReturn(sqlTemplate); + ISqlTemplate sqlTemplateDirty = mock(ISqlTemplate.class); + when(platform.getSqlTemplateDirty()).thenReturn(sqlTemplateDirty); + when(symmetricDialect.getPlatform()).thenReturn(platform); + when(engine.getSymmetricDialect()).thenReturn(symmetricDialect); + IExtensionService extensionService = mock(IExtensionService.class); + when(engine.getExtensionService()).thenReturn(extensionService); + ICacheManager cacheManager = mock(ICacheManager.class); + when(engine.getCacheManager()).thenReturn(cacheManager); + stagingManager = mock(IStagingManager.class); + when(engine.getStagingManager()).thenReturn(stagingManager); + outgoingBatchService = mock(IOutgoingBatchService.class); + when(engine.getOutgoingBatchService()).thenReturn(outgoingBatchService); + IConfigurationService configurationService = mock(IConfigurationService.class); + when(configurationService.getChannel(anyString())).thenReturn(new Channel()); + when(engine.getConfigurationService()).thenReturn(configurationService); + DataExtractorService dataExtractorService = mock(DataExtractorService.class); + when(engine.getDataExtractorService()).thenReturn(dataExtractorService); + IStatisticManager statisticManager = mock(IStatisticManager.class); + when(engine.getStatisticManager()).thenReturn(statisticManager); + INodeService nodeService = mock(INodeService.class); + when(nodeService.findIdentityNodeId()).thenReturn("localNode"); + when(engine.getNodeService()).thenReturn(nodeService); + ITransportManager transportManager = mock(ITransportManager.class); + resumeCache = mock(IHttpResumeCache.class); + when(transportManager.getResumeCache()).thenReturn(resumeCache); + when(engine.getTransportManager()).thenReturn(transportManager); + fileSyncService = spy(new FileSyncService(engine)); + } + + @Test + void getStagedResource_nullBatch_returnsNull() { + assertNull(fileSyncService.getStagedResource(null)); + } + + private static byte[] emptyZipBytes() throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (ZipOutputStream zos = new ZipOutputStream(baos)) { + // no entries: a minimal valid empty zip archive + } + return baos.toByteArray(); + } + + private Node targetNode(String version) { + Node node = new Node(); + node.setNodeId("node1"); + node.setSymmetricVersion(version); + return node; + } + + @Test + void sendFilesForPull_noBatchesAvailable_returnsEmptyResult() { + Node targetNode = targetNode("3.18.0"); + doReturn(new ArrayList()).when(fileSyncService).getBatchesToProcess(targetNode); + FileSyncPullResult result = fileSyncService.prepareFilesForPull(new ProcessInfo(), targetNode, null, null, null); + assertTrue(result.getBatches().isEmpty()); + assertFalse(result.isEnvelopeFormatUsed()); + assertNull(result.getResumeEtag()); + } + + @Test + void sendFilesForPull_resumeDisabled_batchIdParamIsIgnored() { + when(parameterService.is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED)).thenReturn(false); + Node targetNode = targetNode("3.18.0"); + doReturn(new ArrayList()).when(fileSyncService).getBatchesToProcess(targetNode); + FileSyncPullResult result = fileSyncService.prepareFilesForPull(new ProcessInfo(), targetNode, "42", null, null); + assertTrue(result.getBatches().isEmpty()); + assertNull(result.getResumeEtag()); + verify(outgoingBatchService, never()).findOutgoingBatch(anyLong(), anyString()); + } + + @Test + void sendFilesForPull_resumeRequestedButBatchNotFound_fallsBackToNormalPull() { + when(parameterService.is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED)).thenReturn(true); + Node targetNode = targetNode("3.18.0"); + when(outgoingBatchService.findOutgoingBatch(42L, "node1")).thenReturn(null); + doReturn(new ArrayList()).when(fileSyncService).getBatchesToProcess(targetNode); + FileSyncPullResult result = fileSyncService.prepareFilesForPull(new ProcessInfo(), targetNode, "42", null, null); + assertTrue(result.getBatches().isEmpty()); + assertNull(result.getResumeEtag()); + } + + @Test + void sendFilesForPull_resumeWithMatchingEtagAndRange_servesPartialContentFromSkipOffset() { + when(parameterService.is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED)).thenReturn(true); + Node targetNode = targetNode("3.18.0"); + OutgoingBatch batch = new OutgoingBatch(); + batch.setBatchId(42); + batch.setNodeId("node1"); + when(outgoingBatchService.findOutgoingBatch(42L, "node1")).thenReturn(batch); + byte[] content = "0123456789".getBytes(StandardCharsets.UTF_8); + IStagedResource stagedResource = mock(IStagedResource.class); + when(stagedResource.getState()).thenReturn(State.DONE); + when(stagedResource.isFileResource()).thenReturn(true); + when(stagedResource.getSize()).thenReturn((long) content.length); + when(stagedResource.getGenerationTime()).thenReturn(555L); + when(stagedResource.getInputStream()).thenReturn(new ByteArrayInputStream(content)); + doReturn(stagedResource).when(fileSyncService).getStagedResource(batch); + StagedResourceETag matchingEtag = new StagedResourceETag(555L, content.length); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IOutgoingTransport transport = mock(IOutgoingTransport.class); + when(transport.openStream()).thenReturn(out); + ProcessInfo processInfo = new ProcessInfo(); + FileSyncPullResult result = fileSyncService.prepareFilesForPull(processInfo, targetNode, "42", matchingEtag.toJson(), "bytes=4-"); + fileSyncService.writeFilesForPull(new ProcessInfo(), result, transport); + assertTrue(result.isPartialContent()); + assertEquals(4L, result.getSkipCount()); + assertEquals(content.length, result.getTotalSize()); + assertEquals(matchingEtag, result.getResumeEtag()); + assertEquals(1, processInfo.getTotalBatchCount()); + assertArrayEquals("456789".getBytes(StandardCharsets.UTF_8), out.toByteArray()); + } + + @Test + void sendFilesForPull_resumeWithStaleEtag_servesFullContentNotPartial() { + when(parameterService.is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED)).thenReturn(true); + Node targetNode = targetNode("3.18.0"); + OutgoingBatch batch = new OutgoingBatch(); + batch.setBatchId(42); + batch.setNodeId("node1"); + when(outgoingBatchService.findOutgoingBatch(42L, "node1")).thenReturn(batch); + byte[] content = "0123456789".getBytes(StandardCharsets.UTF_8); + IStagedResource stagedResource = mock(IStagedResource.class); + when(stagedResource.getState()).thenReturn(State.DONE); + when(stagedResource.isFileResource()).thenReturn(true); + when(stagedResource.getSize()).thenReturn((long) content.length); + when(stagedResource.getGenerationTime()).thenReturn(555L); + when(stagedResource.getInputStream()).thenReturn(new ByteArrayInputStream(content)); + doReturn(stagedResource).when(fileSyncService).getStagedResource(batch); + StagedResourceETag staleEtag = new StagedResourceETag(999L, content.length); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IOutgoingTransport transport = mock(IOutgoingTransport.class); + when(transport.openStream()).thenReturn(out); + FileSyncPullResult result = fileSyncService.prepareFilesForPull(new ProcessInfo(), targetNode, "42", staleEtag.toJson(), "bytes=4-"); + fileSyncService.writeFilesForPull(new ProcessInfo(), result, transport); + assertFalse(result.isPartialContent()); + assertEquals(0L, result.getSkipCount()); + assertArrayEquals(content, out.toByteArray()); + } + + @Test + void sendFilesForPull_targetNodeBelowVersionGate_fallsBackToSingleBatchLegacyFormat() { + when(parameterService.is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED)).thenReturn(true); + when(parameterService.getLong(ParameterConstants.TRANSPORT_MAX_BYTES_TO_SYNC)).thenReturn(Long.MAX_VALUE); + Node targetNode = targetNode("3.17.0"); + OutgoingBatch batch1 = new OutgoingBatch(); + batch1.setBatchId(1); + batch1.setNodeId("node1"); + batch1.setChannelId(Constants.CHANNEL_FILESYNC); + OutgoingBatch batch2 = new OutgoingBatch(); + batch2.setBatchId(2); + batch2.setNodeId("node1"); + batch2.setChannelId(Constants.CHANNEL_FILESYNC); + doReturn(Arrays.asList(batch1, batch2)).when(fileSyncService).getBatchesToProcess(targetNode); + byte[] zip1 = "ZIP-ONE".getBytes(StandardCharsets.UTF_8); + IStagedResource resource1 = mock(IStagedResource.class); + when(resource1.getSize()).thenReturn((long) zip1.length); + when(resource1.getGenerationTime()).thenReturn(100L); + when(resource1.getInputStream()).thenReturn(new ByteArrayInputStream(zip1)); + Object[] pathComponents1 = fileSyncService.getStagingPathComponents(batch1); + when(stagingManager.create(pathComponents1[0], pathComponents1[1], pathComponents1[2])).thenReturn(resource1); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IOutgoingTransport transport = mock(IOutgoingTransport.class); + when(transport.openStream()).thenReturn(out); + FileSyncPullResult result = fileSyncService.prepareFilesForPull(new ProcessInfo(), targetNode, null, null, null); + fileSyncService.writeFilesForPull(new ProcessInfo(), result, transport); + assertFalse(result.isEnvelopeFormatUsed()); + assertEquals(1, result.getBatches().size()); + assertArrayEquals(zip1, out.toByteArray()); + } + + @Test + void sendFilesForPull_targetNodeAtVersionGate_bundlesMultipleBatchesWithEnvelope() throws IOException { + when(parameterService.is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED)).thenReturn(true); + when(parameterService.getLong(ParameterConstants.TRANSPORT_MAX_BYTES_TO_SYNC)).thenReturn(Long.MAX_VALUE); + Node targetNode = targetNode("3.18.0"); + OutgoingBatch batch1 = new OutgoingBatch(); + batch1.setBatchId(1); + batch1.setNodeId("node1"); + batch1.setChannelId(Constants.CHANNEL_FILESYNC); + OutgoingBatch batch2 = new OutgoingBatch(); + batch2.setBatchId(2); + batch2.setNodeId("node1"); + batch2.setChannelId(Constants.CHANNEL_FILESYNC); + doReturn(Arrays.asList(batch1, batch2)).when(fileSyncService).getBatchesToProcess(targetNode); + byte[] zip1 = "ZIP-ONE".getBytes(StandardCharsets.UTF_8); + IStagedResource resource1 = mock(IStagedResource.class); + when(resource1.getSize()).thenReturn((long) zip1.length); + when(resource1.getGenerationTime()).thenReturn(100L); + when(resource1.getInputStream()).thenReturn(new ByteArrayInputStream(zip1)); + Object[] pathComponents1 = fileSyncService.getStagingPathComponents(batch1); + when(stagingManager.create(pathComponents1[0], pathComponents1[1], pathComponents1[2])).thenReturn(resource1); + byte[] zip2 = "ZIP-TWO-LONGER".getBytes(StandardCharsets.UTF_8); + IStagedResource resource2 = mock(IStagedResource.class); + when(resource2.getSize()).thenReturn((long) zip2.length); + when(resource2.getGenerationTime()).thenReturn(200L); + when(resource2.getInputStream()).thenReturn(new ByteArrayInputStream(zip2)); + Object[] pathComponents2 = fileSyncService.getStagingPathComponents(batch2); + when(stagingManager.create(pathComponents2[0], pathComponents2[1], pathComponents2[2])).thenReturn(resource2); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IOutgoingTransport transport = mock(IOutgoingTransport.class); + when(transport.openStream()).thenReturn(out); + FileSyncPullResult result = fileSyncService.prepareFilesForPull(new ProcessInfo(), targetNode, null, null, null); + fileSyncService.writeFilesForPull(new ProcessInfo(), result, transport); + assertTrue(result.isEnvelopeFormatUsed()); + assertEquals(2, result.getBatches().size()); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + FileSyncBatchEnvelope header1 = FileSyncBatchEnvelope.readHeader(in); + assertEquals(1L, header1.getBatchId()); + assertEquals(zip1.length, header1.getLength()); + assertArrayEquals(zip1, in.readNBytes(zip1.length)); + FileSyncBatchEnvelope header2 = FileSyncBatchEnvelope.readHeader(in); + assertEquals(2L, header2.getBatchId()); + assertArrayEquals(zip2, in.readNBytes(zip2.length)); + assertNull(FileSyncBatchEnvelope.readHeader(in)); + } + + @Test + void processEnvelopedZip_multipleBatches_stagesUnzipsAndClearsEachIndependently() throws IOException { + String sourceNodeId = "remoteNode"; + byte[] zip1 = emptyZipBytes(); + byte[] zip2 = emptyZipBytes(); + StagedResourceETag etag1 = new StagedResourceETag(1L, zip1.length); + StagedResourceETag etag2 = new StagedResourceETag(2L, zip2.length); + ByteArrayOutputStream envelope = new ByteArrayOutputStream(); + FileSyncBatchEnvelope.writeHeader(envelope, 1L, zip1.length, etag1); + envelope.write(zip1); + FileSyncBatchEnvelope.writeHeader(envelope, 2L, zip2.length, etag2); + envelope.write(zip2); + IStagedResource localResource1 = mock(IStagedResource.class); + ByteArrayOutputStream captured1 = new ByteArrayOutputStream(); + when(localResource1.getOutputStream()).thenReturn(captured1); + when(localResource1.getInputStream()).thenAnswer(inv -> new ByteArrayInputStream(captured1.toByteArray())); + when(stagingManager.create(Constants.STAGING_CATEGORY_INCOMING, sourceNodeId, "1_filesync")).thenReturn(localResource1); + IStagedResource localResource2 = mock(IStagedResource.class); + ByteArrayOutputStream captured2 = new ByteArrayOutputStream(); + when(localResource2.getOutputStream()).thenReturn(captured2); + when(localResource2.getInputStream()).thenAnswer(inv -> new ByteArrayInputStream(captured2.toByteArray())); + when(stagingManager.create(Constants.STAGING_CATEGORY_INCOMING, sourceNodeId, "2_filesync")).thenReturn(localResource2); + List result = fileSyncService.processEnvelopedZip(new ByteArrayInputStream(envelope.toByteArray()), + sourceNodeId, new ProcessInfo()); + assertTrue(result.isEmpty()); + verify(localResource1).setState(State.DONE); + verify(localResource1).delete(); + verify(localResource2).setState(State.DONE); + verify(localResource2).delete(); + verify(resumeCache).remove(sourceNodeId, 1L); + verify(resumeCache).remove(sourceNodeId, 2L); + } + + @Test + void processEnvelopedZip_bodyReadFailureWithResumeEnabled_registersResumeCacheEntryAndKeepsPartial() throws IOException { + String sourceNodeId = "remoteNode"; + byte[] zip1 = emptyZipBytes(); + StagedResourceETag etag1 = new StagedResourceETag(1L, zip1.length); + ByteArrayOutputStream envelope = new ByteArrayOutputStream(); + FileSyncBatchEnvelope.writeHeader(envelope, 1L, zip1.length, etag1); + envelope.write(zip1); + IStagedResource localResource1 = mock(IStagedResource.class); + OutputStream throwingOut = new OutputStream() { + @Override + public void write(int b) throws IOException { + throw new IOException("simulated write failure"); + } + }; + when(localResource1.getOutputStream()).thenReturn(throwingOut); + when(localResource1.getSize()).thenReturn(0L); + when(stagingManager.create(Constants.STAGING_CATEGORY_INCOMING, sourceNodeId, "1_filesync")).thenReturn(localResource1); + when(parameterService.is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED)).thenReturn(true); + InputStream envelopeStream = new ByteArrayInputStream(envelope.toByteArray()); + assertThrows(IOException.class, () -> fileSyncService.processEnvelopedZip(envelopeStream, sourceNodeId, new ProcessInfo())); + verify(localResource1, never()).delete(); + verify(localResource1).close(); + ArgumentCaptor captor = ArgumentCaptor.forClass(ResumeCacheEntry.class); + verify(resumeCache).put(anyString(), anyLong(), captor.capture()); + ResumeCacheEntry captured = captor.getValue(); + assertEquals(sourceNodeId, captured.getNodeId()); + assertEquals(1L, captured.getBatchId()); + assertEquals(etag1, captured.getEtag()); + } + + @Test + void processEnvelopedZip_bodyReadFailureWithResumeDisabled_deletesPartialInstead() throws IOException { + String sourceNodeId = "remoteNode"; + byte[] zip1 = emptyZipBytes(); + StagedResourceETag etag1 = new StagedResourceETag(1L, zip1.length); + ByteArrayOutputStream envelope = new ByteArrayOutputStream(); + FileSyncBatchEnvelope.writeHeader(envelope, 1L, zip1.length, etag1); + envelope.write(zip1); + IStagedResource localResource1 = mock(IStagedResource.class); + OutputStream throwingOut = new OutputStream() { + @Override + public void write(int b) throws IOException { + throw new IOException("simulated write failure"); + } + }; + when(localResource1.getOutputStream()).thenReturn(throwingOut); + when(stagingManager.create(Constants.STAGING_CATEGORY_INCOMING, sourceNodeId, "1_filesync")).thenReturn(localResource1); + when(parameterService.is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED)).thenReturn(false); + InputStream envelopeStream = new ByteArrayInputStream(envelope.toByteArray()); + assertThrows(IOException.class, () -> fileSyncService.processEnvelopedZip(envelopeStream, sourceNodeId, new ProcessInfo())); + verify(localResource1).delete(); + verify(resumeCache, never()).put(anyString(), anyLong(), any()); + } + + @Test + void resumePartialBatch_localResourceInCreateState_appendsUnzipsAndClearsResumeCache() throws IOException { + String sourceNodeId = "remoteNode"; + long batchId = 7L; + StagedResourceETag etag = new StagedResourceETag(111L, 999L); + ResumeCacheEntry pendingResume = ResumeCacheEntry.builder() + .nodeId(sourceNodeId) + .batchId(batchId) + .etag(etag) + .receivedCount(3L) + .cachedAtTime(1000L) + .queue(Constants.QUEUE_DEFAULT) + .build(); + byte[] fullZip = emptyZipBytes(); + int splitAt = fullZip.length / 2; + byte[] existingBytes = Arrays.copyOfRange(fullZip, 0, splitAt); + byte[] continuationBytes = Arrays.copyOfRange(fullZip, splitAt, fullZip.length); + IStagedResource localResource = mock(IStagedResource.class); + when(localResource.getState()).thenReturn(State.CREATE); + ByteArrayOutputStream captured = new ByteArrayOutputStream(); + captured.write(existingBytes); + when(localResource.getOutputStream(true)).thenReturn(captured); + when(localResource.getInputStream()).thenAnswer(inv -> new ByteArrayInputStream(captured.toByteArray())); + when(stagingManager.find(Constants.STAGING_CATEGORY_INCOMING, sourceNodeId, batchId + "_filesync")).thenReturn(localResource); + List result = fileSyncService.resumePartialBatch(new ByteArrayInputStream(continuationBytes), sourceNodeId, + new ProcessInfo(), pendingResume); + assertTrue(result.isEmpty()); + assertArrayEquals(fullZip, captured.toByteArray()); + verify(localResource).setState(State.DONE); + verify(localResource).delete(); + verify(resumeCache).remove(sourceNodeId, batchId); + } + + @Test + void resumePartialBatch_localResourceMissing_returnsEmptyListAndClearsResumeCache() throws IOException { + String sourceNodeId = "remoteNode"; + long batchId = 7L; + StagedResourceETag etag = new StagedResourceETag(111L, 999L); + ResumeCacheEntry pendingResume = ResumeCacheEntry.builder() + .nodeId(sourceNodeId) + .batchId(batchId) + .etag(etag) + .receivedCount(3L) + .cachedAtTime(1000L) + .queue(Constants.QUEUE_DEFAULT) + .build(); + when(stagingManager.find(Constants.STAGING_CATEGORY_INCOMING, sourceNodeId, batchId + "_filesync")).thenReturn(null); + List result = fileSyncService.resumePartialBatch(new ByteArrayInputStream(new byte[0]), sourceNodeId, + new ProcessInfo(), pendingResume); + assertTrue(result.isEmpty()); + verify(resumeCache).remove(sourceNodeId, batchId); + } + + @Test + void resumePartialBatch_localResourceAlreadyFinalized_returnsEmptyListAndClearsResumeCache() throws IOException { + String sourceNodeId = "remoteNode"; + long batchId = 7L; + StagedResourceETag etag = new StagedResourceETag(111L, 999L); + ResumeCacheEntry pendingResume = ResumeCacheEntry.builder() + .nodeId(sourceNodeId) + .batchId(batchId) + .etag(etag) + .receivedCount(3L) + .cachedAtTime(1000L) + .queue(Constants.QUEUE_DEFAULT) + .build(); + IStagedResource localResource = mock(IStagedResource.class); + when(localResource.getState()).thenReturn(State.DONE); + when(stagingManager.find(Constants.STAGING_CATEGORY_INCOMING, sourceNodeId, batchId + "_filesync")).thenReturn(localResource); + List result = fileSyncService.resumePartialBatch(new ByteArrayInputStream(new byte[0]), sourceNodeId, + new ProcessInfo(), pendingResume); + assertTrue(result.isEmpty()); + verify(resumeCache).remove(sourceNodeId, batchId); + } + + @Test + void pullFilesFromNode_noPendingFileSyncResume_pullsNormallyWithNullBatchId() throws IOException { + String nodeId = "node1"; + NodeCommunication nodeCommunication = mock(NodeCommunication.class); + Node remoteNode = new Node(); + remoteNode.setNodeId(nodeId); + when(nodeCommunication.getNodeId()).thenReturn(nodeId); + when(nodeCommunication.getNode()).thenReturn(remoteNode); + RemoteNodeStatus status = new RemoteNodeStatus(nodeId, Constants.CHANNEL_FILESYNC, new HashMap<>()); + Node identity = new Node(); + identity.setNodeId("localNode"); + NodeSecurity security = new NodeSecurity(); + when(resumeCache.getPendingFileSyncEntryForNode(nodeId)).thenReturn(null); + when(engine.getStatisticManager().newProcessInfo(any())).thenReturn(new ProcessInfo()); + IIncomingTransport transport = mock(IIncomingTransport.class); + when(transport.getHeaders()).thenReturn(new HashMap<>()); + when(transport.openStream()).thenReturn(new ByteArrayInputStream(new byte[0])); + ITransportManager transportManager = engine.getTransportManager(); + when(transportManager.getFilePullTransport(any(), any(), any(), any(), any(), any())).thenReturn(transport); + doReturn(new ArrayList()).when(fileSyncService).processZip(any(), any(), any()); + fileSyncService.pullFilesFromNode(nodeCommunication, status, identity, security); + @SuppressWarnings("unchecked") + ArgumentCaptor> propsCaptor = ArgumentCaptor.forClass(Map.class); + verify(transportManager).getFilePullTransport(eq(remoteNode), eq(identity), any(), propsCaptor.capture(), any(), isNull()); + assertTrue(propsCaptor.getValue().isEmpty()); + verify(resumeCache, never()).getPendingForNode(anyString(), anyString()); + verify(resumeCache, never()).remove(anyString(), anyLong()); + } + + @Test + void pullFilesFromNode_pendingFileSyncResumeFromDifferentQueue_stillHonoredAndRequestsResumeBatchId() throws IOException { + String nodeId = "node1"; + NodeCommunication nodeCommunication = mock(NodeCommunication.class); + Node remoteNode = new Node(); + remoteNode.setNodeId(nodeId); + when(nodeCommunication.getNodeId()).thenReturn(nodeId); + when(nodeCommunication.getNode()).thenReturn(remoteNode); + RemoteNodeStatus status = new RemoteNodeStatus(nodeId, Constants.CHANNEL_FILESYNC, new HashMap<>()); + Node identity = new Node(); + identity.setNodeId("localNode"); + NodeSecurity security = new NodeSecurity(); + StagedResourceETag etag = new StagedResourceETag(111L, 500L); + ResumeCacheEntry fileSyncResume = ResumeCacheEntry.builder() + .nodeId(nodeId) + .batchId(12L) + .etag(etag) + .receivedCount(200L) + .cachedAtTime(123L) + .queue(Constants.QUEUE_RELOAD) + .fileSync(true) + .build(); + // Registered under a different queue than this attempt's own status.getQueue() - still honored, since file + // sync batch selection is not partitioned by queue the way table-sync's is. + when(resumeCache.getPendingFileSyncEntryForNode(nodeId)).thenReturn(fileSyncResume); + when(engine.getStatisticManager().newProcessInfo(any())).thenReturn(new ProcessInfo()); + IIncomingTransport transport = mock(IIncomingTransport.class); + when(transport.getHeaders()).thenReturn(new HashMap<>()); + when(transport.openStream()).thenReturn(new ByteArrayInputStream(new byte[0])); + ITransportManager transportManager = engine.getTransportManager(); + when(transportManager.getFilePullTransport(any(), any(), any(), any(), any(), any())).thenReturn(transport); + doReturn(new ArrayList()).when(fileSyncService).processZip(any(), any(), any()); + fileSyncService.pullFilesFromNode(nodeCommunication, status, identity, security); + @SuppressWarnings("unchecked") + ArgumentCaptor> propsCaptor = ArgumentCaptor.forClass(Map.class); + verify(transportManager).getFilePullTransport(eq(remoteNode), eq(identity), any(), propsCaptor.capture(), any(), eq(12L)); + assertEquals(etag.toJson(), propsCaptor.getValue().get(WebConstants.HEADER_IF_ETAG)); + assertEquals("bytes=200-", propsCaptor.getValue().get(WebConstants.HEADER_RANGE)); + } + @Test void sendFiles_failureDuringSendPhase_incrementsDataSentErrorsOnly() throws Exception { IStatisticManager statisticManager = mock(IStatisticManager.class); diff --git a/symmetric-core/src/test/java/org/jumpmind/symmetric/transport/HybridTransportManagerTest.java b/symmetric-core/src/test/java/org/jumpmind/symmetric/transport/HybridTransportManagerTest.java index fd765eb3b7..d6428696a1 100644 --- a/symmetric-core/src/test/java/org/jumpmind/symmetric/transport/HybridTransportManagerTest.java +++ b/symmetric-core/src/test/java/org/jumpmind/symmetric/transport/HybridTransportManagerTest.java @@ -45,6 +45,7 @@ import org.jumpmind.symmetric.model.Node; import org.jumpmind.symmetric.service.IParameterService; import org.jumpmind.symmetric.transport.http.HttpTransportManager; +import org.jumpmind.symmetric.transport.http.IHttpResumeCache; import org.jumpmind.symmetric.transport.internal.InternalTransportManager; import org.jumpmind.symmetric.web.WebConstants; import org.junit.jupiter.api.AfterEach; @@ -197,6 +198,34 @@ void testGetPullTransport_delegatesToInternalTransport() throws IOException { verify(internalTransport).getPullTransport(remoteNode, localNode, "token", requestProps, "http://reg"); } + @Test + void testGetPullTransport_sixArg_delegatesToHttpTransport() throws IOException { + IIncomingTransport expectedTransport = mock(IIncomingTransport.class); + Map requestProps = new HashMap(); + when(httpTransport.getPullTransport(remoteNode, localNode, "token", requestProps, "http://reg", 42L)).thenReturn(expectedTransport); + IIncomingTransport result = manager.getPullTransport(remoteNode, localNode, "token", requestProps, "http://reg", 42L); + assertSame(expectedTransport, result); + verify(httpTransport).getPullTransport(remoteNode, localNode, "token", requestProps, "http://reg", 42L); + } + + @Test + void testGetPullTransport_sixArg_delegatesToInternalTransport() throws IOException { + registerEngineForInternalTransport(); + IIncomingTransport expectedTransport = mock(IIncomingTransport.class); + Map requestProps = new HashMap(); + when(internalTransport.getPullTransport(remoteNode, localNode, "token", requestProps, "http://reg", 42L)).thenReturn(expectedTransport); + IIncomingTransport result = manager.getPullTransport(remoteNode, localNode, "token", requestProps, "http://reg", 42L); + assertSame(expectedTransport, result); + verify(internalTransport).getPullTransport(remoteNode, localNode, "token", requestProps, "http://reg", 42L); + } + + @Test + void testGetResumeCache_delegatesToHttpTransport() { + IHttpResumeCache expectedCache = mock(IHttpResumeCache.class); + when(httpTransport.getResumeCache()).thenReturn(expectedCache); + assertSame(expectedCache, manager.getResumeCache()); + } + @Test void testGetPushTransport_delegatesToHttpTransport() throws IOException { IOutgoingWithResponseTransport expectedTransport = mock(IOutgoingWithResponseTransport.class); diff --git a/symmetric-core/src/test/java/org/jumpmind/symmetric/transport/http/DefaultHttpResumeCacheTest.java b/symmetric-core/src/test/java/org/jumpmind/symmetric/transport/http/DefaultHttpResumeCacheTest.java new file mode 100644 index 0000000000..2fa04ab0c7 --- /dev/null +++ b/symmetric-core/src/test/java/org/jumpmind/symmetric/transport/http/DefaultHttpResumeCacheTest.java @@ -0,0 +1,174 @@ +/** + * Licensed to JumpMind Inc under one or more contributor + * license agreements. See the NOTICE file distributed + * with this work for additional information regarding + * copyright ownership. JumpMind Inc licenses this file + * to you under the GNU Affero General Public License, version 3.0 (AGPLv3) + * (the "License"); you may not use this file except in compliance + * with the License. + * + * You should have received a copy of the GNU Affero General Public License, + * version 3.0 (AGPLv3) along with this library; if not, see + * . + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jumpmind.symmetric.transport.http; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import org.jumpmind.symmetric.common.Constants; +import org.jumpmind.symmetric.io.stage.StagedResourceETag; +import org.junit.jupiter.api.Test; + +class DefaultHttpResumeCacheTest { + @Test + void testPutAndGet_returnsStoredEntry() { + DefaultHttpResumeCache cache = new DefaultHttpResumeCache(null); + ResumeCacheEntry entry = createEntry("node1", 100L); + cache.put("node1", 100L, entry); + assertEquals(entry, cache.get("node1", 100L)); + } + + @Test + void testGet_withWrongBatchId_returnsNull() { + DefaultHttpResumeCache cache = new DefaultHttpResumeCache(null); + cache.put("node1", 100L, createEntry("node1", 100L)); + assertNull(cache.get("node1", 101L)); + } + + @Test + void testGet_withWrongNodeId_returnsNull() { + DefaultHttpResumeCache cache = new DefaultHttpResumeCache(null); + cache.put("node1", 100L, createEntry("node1", 100L)); + assertNull(cache.get("node2", 100L)); + } + + @Test + void testGet_whenEmpty_returnsNull() { + DefaultHttpResumeCache cache = new DefaultHttpResumeCache(null); + assertNull(cache.get("node1", 100L)); + } + + @Test + void testGetPendingForNode_returnsEntryForMatchingNode() { + DefaultHttpResumeCache cache = new DefaultHttpResumeCache(null); + ResumeCacheEntry entry = createEntry("node1", 100L); + cache.put("node1", 100L, entry); + assertEquals(entry, cache.getPendingForNode("node1", Constants.QUEUE_DEFAULT)); + } + + @Test + void testGetPendingForNode_withNoMatch_returnsNull() { + DefaultHttpResumeCache cache = new DefaultHttpResumeCache(null); + cache.put("node1", 100L, createEntry("node1", 100L)); + assertNull(cache.getPendingForNode("node2", Constants.QUEUE_DEFAULT)); + } + + @Test + void testGetPendingForNode_withMismatchedQueue_returnsNull() { + DefaultHttpResumeCache cache = new DefaultHttpResumeCache(null); + cache.put("node1", 100L, createEntry("node1", 100L)); + assertNull(cache.getPendingForNode("node1", Constants.QUEUE_SYSTEM)); + } + + @Test + void testGetPendingFileSyncEntryForNode_returnsEntryIgnoringQueue() { + DefaultHttpResumeCache cache = new DefaultHttpResumeCache(null); + ResumeCacheEntry entry = createFileSyncEntry("node1", 100L, Constants.QUEUE_RELOAD); + cache.put("node1", 100L, entry); + assertEquals(entry, cache.getPendingFileSyncEntryForNode("node1")); + } + + @Test + void testGetPendingFileSyncEntryForNode_withTableSyncEntry_returnsNull() { + DefaultHttpResumeCache cache = new DefaultHttpResumeCache(null); + cache.put("node1", 100L, createEntry("node1", 100L)); + assertNull(cache.getPendingFileSyncEntryForNode("node1")); + } + + @Test + void testGetPendingFileSyncEntryForNode_withWrongNodeId_returnsNull() { + DefaultHttpResumeCache cache = new DefaultHttpResumeCache(null); + cache.put("node1", 100L, createFileSyncEntry("node1", 100L, Constants.QUEUE_DEFAULT)); + assertNull(cache.getPendingFileSyncEntryForNode("node2")); + } + + @Test + void testRemove_withMatchingKey_clearsEntry() { + DefaultHttpResumeCache cache = new DefaultHttpResumeCache(null); + cache.put("node1", 100L, createEntry("node1", 100L)); + cache.remove("node1", 100L); + assertNull(cache.get("node1", 100L)); + assertNull(cache.getPendingForNode("node1", Constants.QUEUE_DEFAULT)); + } + + @Test + void testRemove_withNonMatchingKey_leavesEntryInPlace() { + DefaultHttpResumeCache cache = new DefaultHttpResumeCache(null); + ResumeCacheEntry entry = createEntry("node1", 100L); + cache.put("node1", 100L, entry); + cache.remove("node1", 999L); + assertEquals(entry, cache.get("node1", 100L)); + } + + @Test + void testPut_differentOwnerWhileSlotBusy_doesNotEvictExistingEntry() { + DefaultHttpResumeCache cache = new DefaultHttpResumeCache(null); + ResumeCacheEntry first = createEntry("node1", 100L); + cache.put("node1", 100L, first); + cache.put("node2", 200L, createEntry("node2", 200L)); + assertEquals(first, cache.get("node1", 100L)); + assertNull(cache.get("node2", 200L)); + } + + @Test + void testPut_sameOwnerReRegistering_replacesEntry() { + DefaultHttpResumeCache cache = new DefaultHttpResumeCache(null); + cache.put("node1", 100L, createEntry("node1", 100L)); + ResumeCacheEntry updated = createEntry("node1", 100L); + cache.put("node1", 100L, updated); + assertEquals(updated, cache.get("node1", 100L)); + } + + @Test + void testPut_afterRemove_slotAcceptsNewOwner() { + DefaultHttpResumeCache cache = new DefaultHttpResumeCache(null); + cache.put("node1", 100L, createEntry("node1", 100L)); + cache.remove("node1", 100L); + ResumeCacheEntry second = createEntry("node2", 200L); + cache.put("node2", 200L, second); + assertEquals(second, cache.get("node2", 200L)); + } + + private ResumeCacheEntry createEntry(String nodeId, long batchId) { + return ResumeCacheEntry.builder() + .nodeId(nodeId) + .batchId(batchId) + .etag(new StagedResourceETag(1L, 2L)) + .receivedCount(3L) + .channelId("channel1") + .binaryEncoding("NONE") + .cachedAtTime(4L) + .queue(Constants.QUEUE_DEFAULT) + .build(); + } + + private ResumeCacheEntry createFileSyncEntry(String nodeId, long batchId, String queue) { + return ResumeCacheEntry.builder() + .nodeId(nodeId) + .batchId(batchId) + .etag(new StagedResourceETag(1L, 2L)) + .receivedCount(3L) + .cachedAtTime(4L) + .queue(queue) + .fileSync(true) + .build(); + } +} diff --git a/symmetric-core/src/test/java/org/jumpmind/symmetric/transport/http/HttpIncomingTransportTest.java b/symmetric-core/src/test/java/org/jumpmind/symmetric/transport/http/HttpIncomingTransportTest.java new file mode 100644 index 0000000000..183d5693d6 --- /dev/null +++ b/symmetric-core/src/test/java/org/jumpmind/symmetric/transport/http/HttpIncomingTransportTest.java @@ -0,0 +1,98 @@ +/** + * Licensed to JumpMind Inc under one or more contributor + * license agreements. See the NOTICE file distributed + * with this work for additional information regarding + * copyright ownership. JumpMind Inc licenses this file + * to you under the GNU Affero General Public License, version 3.0 (AGPLv3) + * (the "License"); you may not use this file except in compliance + * with the License. + * + * You should have received a copy of the GNU Affero General Public License, + * version 3.0 (AGPLv3) along with this library; if not, see + * . + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jumpmind.symmetric.transport.http; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; + +import org.jumpmind.exception.HttpException; +import org.jumpmind.symmetric.common.ParameterConstants; +import org.jumpmind.symmetric.service.IParameterService; +import org.jumpmind.symmetric.service.RegistrationRequiredException; +import org.jumpmind.symmetric.web.WebConstants; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class HttpIncomingTransportTest { + private HttpTransportManager httpTransportManager; + private HttpConnection connection; + private IParameterService parameterService; + + @BeforeEach + void setUp() throws IOException { + httpTransportManager = mock(HttpTransportManager.class); + connection = mock(HttpConnection.class); + parameterService = mock(IParameterService.class); + when(parameterService.is(eq(ParameterConstants.TRANSPORT_HTTP_MANUAL_REDIRECTS_ENABLED), anyBoolean())).thenReturn(false); + when(parameterService.getInt(ParameterConstants.TRANSPORT_HTTP_TIMEOUT)).thenReturn(30000); + when(connection.getContentEncoding()).thenReturn(null); + when(connection.getInputStream()).thenReturn(new ByteArrayInputStream(new byte[] { 1, 2, 3 })); + } + + @Test + void testOpenStream_withOk_returnsInputStream() throws IOException { + when(connection.getResponseCode()).thenReturn(WebConstants.SC_OK); + HttpIncomingTransport transport = new HttpIncomingTransport(httpTransportManager, connection, parameterService); + InputStream result = transport.openStream(); + assertNotNull(result); + } + + @Test + void testOpenStream_withPartialContent_returnsInputStream() throws IOException { + when(connection.getResponseCode()).thenReturn(WebConstants.SC_PARTIAL_CONTENT); + HttpIncomingTransport transport = new HttpIncomingTransport(httpTransportManager, connection, parameterService); + InputStream result = transport.openStream(); + assertNotNull(result); + } + + @Test + void testOpenStream_withPartialContent_updatesSessionLikeOk() throws IOException { + when(connection.getResponseCode()).thenReturn(WebConstants.SC_PARTIAL_CONTENT); + HttpIncomingTransport transport = new HttpIncomingTransport(httpTransportManager, connection, parameterService); + transport.openStream(); + verify(httpTransportManager).updateSession(connection); + } + + @Test + void testOpenStream_withRegistrationRequired_throwsRegistrationRequiredException() throws IOException { + when(connection.getResponseCode()).thenReturn(WebConstants.REGISTRATION_REQUIRED); + HttpIncomingTransport transport = new HttpIncomingTransport(httpTransportManager, connection, parameterService); + assertThrows(RegistrationRequiredException.class, transport::openStream); + } + + @Test + void testOpenStream_withUnrecognizedCode_throwsHttpException() throws IOException { + when(connection.getResponseCode()).thenReturn(599); + HttpIncomingTransport transport = new HttpIncomingTransport(httpTransportManager, connection, parameterService); + HttpException ex = assertThrows(HttpException.class, transport::openStream); + assertEquals(599, ex.getCode()); + } +} diff --git a/symmetric-core/src/test/java/org/jumpmind/symmetric/transport/http/HttpTransportManagerTest.java b/symmetric-core/src/test/java/org/jumpmind/symmetric/transport/http/HttpTransportManagerTest.java index f6ccd229b6..78d5951558 100644 --- a/symmetric-core/src/test/java/org/jumpmind/symmetric/transport/http/HttpTransportManagerTest.java +++ b/symmetric-core/src/test/java/org/jumpmind/symmetric/transport/http/HttpTransportManagerTest.java @@ -1,6 +1,10 @@ package org.jumpmind.symmetric.transport.http; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.ArgumentMatchers.anyString; @@ -12,6 +16,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.net.URL; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -20,9 +25,11 @@ import org.jumpmind.symmetric.common.ParameterConstants; import org.jumpmind.symmetric.model.IncomingBatch; import org.jumpmind.symmetric.model.Node; +import org.jumpmind.symmetric.service.IExtensionService; import org.jumpmind.symmetric.service.IParameterService; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; public class HttpTransportManagerTest { private HttpTransportManager manager; @@ -39,10 +46,14 @@ void setUp() throws Exception { localNode = mock(Node.class); batch = mock(IncomingBatch.class); ps = mock(IParameterService.class); + IExtensionService extensionService = mock(IExtensionService.class); + when(extensionService.getExtensionPointList(any())).thenReturn(Collections.emptyList()); + when(engine.getExtensionService()).thenReturn(extensionService); when(remoteNode.getNodeId()).thenReturn("remote-001"); when(localNode.getNodeId()).thenReturn("local-001"); when(remoteNode.getNodeGroupId()).thenReturn("group-remote"); when(localNode.getNodeGroupId()).thenReturn("group-local"); + when(remoteNode.getSyncUrl()).thenReturn("http://remote.example/sync"); when(engine.getParameterService()).thenReturn(ps); when(engine.getParameterService().getInt(ParameterConstants.TRANSPORT_MAX_FORM_KEYS)).thenReturn(1000); when(engine.getParameterService().getInt(ParameterConstants.TRANSPORT_MAX_BYTES_TO_SYNC)).thenReturn(100000); @@ -53,6 +64,12 @@ void setUp() throws Exception { anyString(), anyString(), anyMap(), anyString()); } + @Test + void testConstructor_initializesDefaultResumeCache() { + assertNotNull(manager.getResumeCache()); + assertInstanceOf(DefaultHttpResumeCache.class, manager.getResumeCache()); + } + @Test void testSendAcknowledgement_basic() throws Exception { List batches = List.of(batch); @@ -96,4 +113,34 @@ void testSendAcknowledgement_setsDefaultMaxFormKeys_whenBackOffAndZeroMaxFormKey int result = manager.sendAcknowledgement(remoteNode, batches, localNode, "token", new HashMap<>(), "http://url"); assertEquals(200, result); } + + @Test + void testGetPullTransport_sixArgWithResumeBatchId_appendsBatchIdToUrl() throws Exception { + when(remoteNode.getSymmetricVersion()).thenReturn("3.18.0"); + HttpConnection conn = mock(HttpConnection.class); + ArgumentCaptor urlCaptor = ArgumentCaptor.forClass(URL.class); + doReturn(conn).when(manager).createGetConnectionFor(urlCaptor.capture(), anyString(), any()); + manager.getPullTransport(remoteNode, localNode, "token", new HashMap<>(), "http://reg", 42L); + assertTrue(urlCaptor.getValue().toString().contains("batchId=42")); + } + + @Test + void testGetPullTransport_sixArgWithNullResumeBatchId_omitsBatchIdFromUrl() throws Exception { + when(remoteNode.getSymmetricVersion()).thenReturn("3.18.0"); + HttpConnection conn = mock(HttpConnection.class); + ArgumentCaptor urlCaptor = ArgumentCaptor.forClass(URL.class); + doReturn(conn).when(manager).createGetConnectionFor(urlCaptor.capture(), anyString(), any()); + manager.getPullTransport(remoteNode, localNode, "token", new HashMap<>(), "http://reg", null); + assertFalse(urlCaptor.getValue().toString().contains("batchId=")); + } + + @Test + void testGetPullTransport_fiveArg_delegatesWithoutResumeBatchId() throws Exception { + when(remoteNode.getSymmetricVersion()).thenReturn("3.18.0"); + HttpConnection conn = mock(HttpConnection.class); + ArgumentCaptor urlCaptor = ArgumentCaptor.forClass(URL.class); + doReturn(conn).when(manager).createGetConnectionFor(urlCaptor.capture(), anyString(), any()); + manager.getPullTransport(remoteNode, localNode, "token", new HashMap<>(), "http://reg"); + assertFalse(urlCaptor.getValue().toString().contains("batchId=")); + } } diff --git a/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/CsvConstants.java b/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/CsvConstants.java index b69cf416e2..460a9f260b 100644 --- a/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/CsvConstants.java +++ b/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/CsvConstants.java @@ -48,4 +48,5 @@ private CsvConstants() { public static final String STATS_COLUMNS = "stats_columns"; public static final String BASETIME = "basetime"; public static final String TIME = "ts"; + public static final String ETAG = "etag"; } diff --git a/symmetric-io/src/main/java/org/jumpmind/symmetric/io/stage/IStagedResource.java b/symmetric-io/src/main/java/org/jumpmind/symmetric/io/stage/IStagedResource.java index 27e88b2cb5..4d377915d6 100644 --- a/symmetric-io/src/main/java/org/jumpmind/symmetric/io/stage/IStagedResource.java +++ b/symmetric-io/src/main/java/org/jumpmind/symmetric/io/stage/IStagedResource.java @@ -39,6 +39,10 @@ public String getExtensionName() { public BufferedWriter getWriter(long threshold); + default BufferedWriter getWriter(long threshold, boolean append) { + return getWriter(threshold); + } + public OutputStream getOutputStream(); public OutputStream getOutputStream(boolean append); @@ -63,6 +67,10 @@ public String getExtensionName() { public void refreshLastUpdateTime(); + default long getGenerationTime() { + return getLastUpdateTime(); + } + public boolean isFileResource(); public boolean isMemoryResource(); diff --git a/symmetric-io/src/main/java/org/jumpmind/symmetric/io/stage/StagedResource.java b/symmetric-io/src/main/java/org/jumpmind/symmetric/io/stage/StagedResource.java index 86301fc819..14543b295c 100644 --- a/symmetric-io/src/main/java/org/jumpmind/symmetric/io/stage/StagedResource.java +++ b/symmetric-io/src/main/java/org/jumpmind/symmetric/io/stage/StagedResource.java @@ -53,6 +53,7 @@ public class StagedResource implements IStagedResource { protected String path; protected StringBuilder memoryBuffer; protected long lastUpdateTime; + protected long generationTime; protected State state; protected OutputStream outputStream = null; protected Map inputStreams = null; @@ -77,6 +78,7 @@ public StagedResource(File directory, String path, StagingManager stagingManager lastUpdateTime = file.lastModified(); } } + this.generationTime = lastUpdateTime; } protected static String toPath(File directory, File file) { @@ -163,6 +165,11 @@ public void setState(State state) { refreshLastUpdateTime(); this.state = state; this.file = buildFile(state); + if (state == State.DONE && this.file.exists() && !this.file.setLastModified(generationTime)) { + log.warn( + "Failed to pin the last-modified time of '{}' to its generation time; a future re-lookup of this resource may derive a different generation time", + this.file.getAbsolutePath()); + } } protected void handleFailedRename(File oldFile, File newFile) { @@ -367,23 +374,36 @@ protected InputStream createInputStream() throws FileNotFoundException { } public BufferedWriter getWriter(long threshold) { + return getWriter(threshold, false); + } + + @Override + public BufferedWriter getWriter(long threshold, boolean append) { refreshLastUpdateTime(); if (writer == null) { - if (file != null && file.exists()) { - log.warn("getWriter had to delete {} because it already existed.", file.getAbsolutePath()); - file.delete(); - } else if (this.memoryBuffer != null) { - log.warn("We had to delete the memory buffer for {} because it already existed", getPath()); + if (!append) { + if (file != null && file.exists()) { + log.warn("getWriter had to delete {} because it already existed.", file.getAbsolutePath()); + file.delete(); + } else if (this.memoryBuffer != null) { + log.warn("We had to delete the memory buffer for {} because it already existed", getPath()); + this.memoryBuffer = null; + } + this.memoryBuffer = threshold > 0 ? new StringBuilder() : null; + } else { this.memoryBuffer = null; } - this.memoryBuffer = threshold > 0 ? new StringBuilder() : null; - writer = createWriter(threshold); + writer = createWriter(threshold, append); } return writer; } protected BufferedWriter createWriter(long threshold) { - return new BufferedWriter(new ThresholdFileWriter(threshold, this.memoryBuffer, file)); + return createWriter(threshold, false); + } + + protected BufferedWriter createWriter(long threshold, boolean append) { + return new BufferedWriter(new ThresholdFileWriter(threshold, this.memoryBuffer, file, append)); } public long getSize() { @@ -408,6 +428,20 @@ public void refreshLastUpdateTime() { this.lastUpdateTime = System.currentTimeMillis(); } + @Override + public long getGenerationTime() { + return generationTime; + } + + /** + * Stamps {@code generationTime} to now, discarding whatever value the constructor derived from a stale file that previously occupied this path. Called by + * {@link StagingManager#create(Object...)} after it deletes such a file, so a freshly-created resource's identity doesn't get inherited from unrelated + * prior content. + */ + void resetGenerationTime() { + this.generationTime = System.currentTimeMillis(); + } + public boolean delete() { close(); boolean deleted = false; diff --git a/symmetric-io/src/main/java/org/jumpmind/symmetric/io/stage/StagedResourceETag.java b/symmetric-io/src/main/java/org/jumpmind/symmetric/io/stage/StagedResourceETag.java new file mode 100644 index 0000000000..bc74e950fc --- /dev/null +++ b/symmetric-io/src/main/java/org/jumpmind/symmetric/io/stage/StagedResourceETag.java @@ -0,0 +1,119 @@ +/** + * Licensed to JumpMind Inc under one or more contributor + * license agreements. See the NOTICE file distributed + * with this work for additional information regarding + * copyright ownership. JumpMind Inc licenses this file + * to you under the GNU Affero General Public License, version 3.0 (AGPLv3) + * (the "License"); you may not use this file except in compliance + * with the License. + * + * You should have received a copy of the GNU Affero General Public License, + * version 3.0 (AGPLv3) along with this library; if not, see + * . + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jumpmind.symmetric.io.stage; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.gson.Gson; +import com.google.gson.JsonSyntaxException; + +/** + * Identifies a specific version of a staged resource's content, so a resumed transfer can tell whether a partially-received copy is still valid to append to, + * or stale and needing a full resend. Based on the resource's stable {@link IStagedResource#getGenerationTime()}, not + * {@link IStagedResource#getLastUpdateTime()}, which is refreshed on every access and would make every ETag comparison a false mismatch. + */ +public class StagedResourceETag { + public static final int CURRENT_VERSION = 1; + private static final Logger log = LoggerFactory.getLogger(StagedResourceETag.class); + private static final Gson GSON = new Gson(); + private int version = CURRENT_VERSION; + private long generationTime; + private long size; + + public StagedResourceETag() { + } + + public StagedResourceETag(long generationTime, long size) { + this.generationTime = generationTime; + this.size = size; + } + + public int getVersion() { + return version; + } + + public void setVersion(int version) { + this.version = version; + } + + public long getGenerationTime() { + return generationTime; + } + + public void setGenerationTime(long generationTime) { + this.generationTime = generationTime; + } + + public long getSize() { + return size; + } + + public void setSize(long size) { + this.size = size; + } + + public String toJson() { + return GSON.toJson(this); + } + + /** + * Never throws. A stale, malformed, or future-incompatible {@code If-ETag} value should be treated as "no match, do a full resend" rather than fail the + * request. + */ + public static StagedResourceETag fromJson(String json) { + if (json == null || json.isEmpty()) { + return null; + } + try { + StagedResourceETag etag = GSON.fromJson(json, StagedResourceETag.class); + if (etag == null || etag.version != CURRENT_VERSION) { + return null; + } + return etag; + } catch (JsonSyntaxException e) { + log.debug("Ignoring unparseable staged resource ETag: {}", json, e); + return null; + } + } + + @Override + public int hashCode() { + return Long.hashCode(version * 31L + generationTime * 31L + size); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof StagedResourceETag)) { + return false; + } + StagedResourceETag other = (StagedResourceETag) obj; + return version == other.version && generationTime == other.generationTime && size == other.size; + } + + @Override + public String toString() { + return toJson(); + } +} diff --git a/symmetric-io/src/main/java/org/jumpmind/symmetric/io/stage/StagingManager.java b/symmetric-io/src/main/java/org/jumpmind/symmetric/io/stage/StagingManager.java index 45bb18bc0c..df4ae0ee00 100644 --- a/symmetric-io/src/main/java/org/jumpmind/symmetric/io/stage/StagingManager.java +++ b/symmetric-io/src/main/java/org/jumpmind/symmetric/io/stage/StagingManager.java @@ -197,6 +197,7 @@ public IStagedResource create(Object... path) { IStagedResource resource = createStagedResource(filePath); if (resource.exists()) { resource.delete(); + ((StagedResource) resource).resetGenerationTime(); } else { resource.getFile().getParentFile().mkdirs(); } diff --git a/symmetric-io/src/main/java/org/jumpmind/symmetric/io/stage/ThresholdFileWriter.java b/symmetric-io/src/main/java/org/jumpmind/symmetric/io/stage/ThresholdFileWriter.java index 5f07620e05..0c5dbb0e43 100644 --- a/symmetric-io/src/main/java/org/jumpmind/symmetric/io/stage/ThresholdFileWriter.java +++ b/symmetric-io/src/main/java/org/jumpmind/symmetric/io/stage/ThresholdFileWriter.java @@ -41,6 +41,7 @@ public class ThresholdFileWriter extends Writer { protected BufferedWriter fileWriter; protected StringBuilder buffer; protected long threshhold; + protected boolean append; /** * @param threshold @@ -49,9 +50,22 @@ public class ThresholdFileWriter extends Writer { * The file to write to after the threshold has been reached */ public ThresholdFileWriter(long threshold, StringBuilder buffer, File file) { + this(threshold, buffer, file, false); + } + + /** + * @param threshold + * The number of bytes at which to start writing to a file + * @param file + * The file to write to after the threshold has been reached + * @param append + * When true, write to the end of an existing file instead of truncating it + */ + public ThresholdFileWriter(long threshold, StringBuilder buffer, File file, boolean append) { this.file = file; this.buffer = buffer; this.threshhold = threshold; + this.append = append; } public File getFile() { @@ -97,7 +111,7 @@ public void write(char[] cbuf, int off, int len) throws IOException { } protected BufferedWriter getWriter() throws IOException { - return new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), StandardCharsets.UTF_8.name())); + return new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file, append), StandardCharsets.UTF_8.name())); } public BufferedReader getReader() throws IOException { diff --git a/symmetric-io/src/test/java/org/jumpmind/symmetric/io/data/stage/ThresholdFileWriterTest.java b/symmetric-io/src/test/java/org/jumpmind/symmetric/io/data/stage/ThresholdFileWriterTest.java index 3d57987481..546fa353ed 100644 --- a/symmetric-io/src/test/java/org/jumpmind/symmetric/io/data/stage/ThresholdFileWriterTest.java +++ b/symmetric-io/src/test/java/org/jumpmind/symmetric/io/data/stage/ThresholdFileWriterTest.java @@ -25,12 +25,57 @@ import org.apache.commons.io.IOUtils; import org.jumpmind.symmetric.io.stage.ThresholdFileWriter; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.*; import org.junit.jupiter.api.Test; public class ThresholdFileWriterTest { final String TEST_STR = "The quick brown fox jumped over the lazy dog"; + @Test + public void testAppendConstructor_appendsToExistingFileContent() throws Exception { + File file = getTestFile(); + ThresholdFileWriter writer = new ThresholdFileWriter(0, null, file); + writer.write("hello "); + writer.close(); + ThresholdFileWriter appendingWriter = new ThresholdFileWriter(0, null, file, true); + appendingWriter.write("world"); + appendingWriter.close(); + BufferedReader reader = appendingWriter.getReader(); + assertEquals("hello world", IOUtils.toString(reader)); + reader.close(); + assertTrue(file.delete()); + } + + @Test + public void testNonAppendConstructor_stillOverwritesExistingFileContent() throws Exception { + File file = getTestFile(); + ThresholdFileWriter writer = new ThresholdFileWriter(0, null, file); + writer.write("original content"); + writer.close(); + ThresholdFileWriter overwritingWriter = new ThresholdFileWriter(0, null, file, false); + overwritingWriter.write("new"); + overwritingWriter.close(); + BufferedReader reader = overwritingWriter.getReader(); + assertEquals("new", IOUtils.toString(reader)); + reader.close(); + assertTrue(file.delete()); + } + + @Test + public void testThreeArgConstructor_defaultsToNonAppendBehavior() throws Exception { + File file = getTestFile(); + ThresholdFileWriter writer = new ThresholdFileWriter(0, null, file); + writer.write("original content"); + writer.close(); + ThresholdFileWriter secondWriter = new ThresholdFileWriter(0, null, file); + secondWriter.write("new"); + secondWriter.close(); + BufferedReader reader = secondWriter.getReader(); + assertEquals("new", IOUtils.toString(reader)); + reader.close(); + assertTrue(file.delete()); + } + @Test public void testNoWriteToFile() throws Exception { File file = getTestFile(); diff --git a/symmetric-io/src/test/java/org/jumpmind/symmetric/io/stage/StagedResourceETagTest.java b/symmetric-io/src/test/java/org/jumpmind/symmetric/io/stage/StagedResourceETagTest.java new file mode 100644 index 0000000000..a8c3e67fa3 --- /dev/null +++ b/symmetric-io/src/test/java/org/jumpmind/symmetric/io/stage/StagedResourceETagTest.java @@ -0,0 +1,70 @@ +/** + * Licensed to JumpMind Inc under one or more contributor + * license agreements. See the NOTICE file distributed + * with this work for additional information regarding + * copyright ownership. JumpMind Inc licenses this file + * to you under the GNU Affero General Public License, version 3.0 (AGPLv3) + * (the "License"); you may not use this file except in compliance + * with the License. + * + * You should have received a copy of the GNU Affero General Public License, + * version 3.0 (AGPLv3) along with this library; if not, see + * . + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jumpmind.symmetric.io.stage; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import org.junit.jupiter.api.Test; + +class StagedResourceETagTest { + @Test + void testToJsonAndFromJson_roundTrips() { + StagedResourceETag etag = new StagedResourceETag(123456789L, 987654321L); + StagedResourceETag roundTripped = StagedResourceETag.fromJson(etag.toJson()); + assertEquals(etag, roundTripped); + assertEquals(etag.getGenerationTime(), roundTripped.getGenerationTime()); + assertEquals(etag.getSize(), roundTripped.getSize()); + assertEquals(StagedResourceETag.CURRENT_VERSION, roundTripped.getVersion()); + } + + @Test + void testFromJson_withNull_returnsNull() { + assertNull(StagedResourceETag.fromJson(null)); + } + + @Test + void testFromJson_withEmptyString_returnsNull() { + assertNull(StagedResourceETag.fromJson("")); + } + + @Test + void testFromJson_withMalformedJson_returnsNull() { + assertNull(StagedResourceETag.fromJson("{not valid json")); + } + + @Test + void testFromJson_withFutureVersion_returnsNull() { + String json = "{\"version\":" + (StagedResourceETag.CURRENT_VERSION + 1) + ",\"generationTime\":1,\"size\":2}"; + assertNull(StagedResourceETag.fromJson(json)); + } + + @Test + void testEquals_differsWhenGenerationTimeOrSizeDiffer() { + StagedResourceETag base = new StagedResourceETag(100L, 200L); + StagedResourceETag differentGenerationTime = new StagedResourceETag(101L, 200L); + StagedResourceETag differentSize = new StagedResourceETag(100L, 201L); + assertEquals(base, new StagedResourceETag(100L, 200L)); + assertNotEquals(base, differentGenerationTime); + assertNotEquals(base, differentSize); + } +} diff --git a/symmetric-io/src/test/java/org/jumpmind/symmetric/io/stage/StagedResourceTest.java b/symmetric-io/src/test/java/org/jumpmind/symmetric/io/stage/StagedResourceTest.java new file mode 100644 index 0000000000..ade46194f5 --- /dev/null +++ b/symmetric-io/src/test/java/org/jumpmind/symmetric/io/stage/StagedResourceTest.java @@ -0,0 +1,166 @@ +/** + * Licensed to JumpMind Inc under one or more contributor + * license agreements. See the NOTICE file distributed + * with this work for additional information regarding + * copyright ownership. JumpMind Inc licenses this file + * to you under the GNU Affero General Public License, version 3.0 (AGPLv3) + * (the "License"); you may not use this file except in compliance + * with the License. + * + * You should have received a copy of the GNU Affero General Public License, + * version 3.0 (AGPLv3) along with this library; if not, see + * . + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jumpmind.symmetric.io.stage; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.IOException; + +import org.apache.commons.io.IOUtils; +import org.jumpmind.symmetric.io.stage.IStagedResource.State; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class StagedResourceTest { + @TempDir + File tempDir; + + @Test + void testGetGenerationTime_isStableAcrossRefreshLastUpdateTime() { + StagingManager manager = newManager(); + StagedResource resource = new StagedResource(tempDir, "path1", manager); + long generationTime = resource.getGenerationTime(); + resource.refreshLastUpdateTime(); + resource.refreshLastUpdateTime(); + assertEquals(generationTime, resource.getGenerationTime()); + } + + @Test + void testGetGenerationTime_forFreshlyLookedUpResource_reflectsFileLastModified() throws IOException { + StagingManager manager = newManager(); + StagedResource original = new StagedResource(tempDir, "path1", manager); + writeAndClose(original, "hello", false); + StagedResource lookedUpAgain = new StagedResource(tempDir, "path1", manager); + assertEquals(original.getFile().lastModified(), lookedUpAgain.getGenerationTime()); + } + + @Test + void testCreate_reusingPathWithStaleLeftoverFile_doesNotInheritStaleGenerationTime() throws IOException { + StagingManager manager = newManager(); + IStagedResource first = manager.create("path1"); + writeAndClose((StagedResource) first, "stale content from a prior attempt", false); + first.getFile().setLastModified(System.currentTimeMillis() - 60000); + long beforeRecreate = System.currentTimeMillis(); + IStagedResource second = manager.create("path1"); + assertTrue(second.getGenerationTime() >= beforeRecreate); + } + + @Test + void testGetGenerationTime_forDoneResource_isStableAcrossReconstructionAfterMultipleWrites() throws IOException { + StagingManager manager = newManager(); + StagedResource original = new StagedResource(tempDir, "path1", manager); + long originalGenerationTime = original.getGenerationTime(); + BufferedWriter writer = original.getWriter(0); + writer.write("first chunk "); + writer.flush(); + assertTrue(original.file.setLastModified(originalGenerationTime - 5000)); + writer.write("second chunk"); + original.setState(State.DONE); + StagedResource reconstructed = new StagedResource(tempDir, "path1", manager); + assertEquals(originalGenerationTime, reconstructed.getGenerationTime()); + } + + @Test + void testGetWriter_nonAppendMode_overwritesExistingContent() throws IOException { + StagingManager manager = newManager(); + StagedResource first = new StagedResource(tempDir, "path1", manager); + writeAndClose(first, "original content", false); + StagedResource second = new StagedResource(tempDir, "path1", manager); + writeAndClose(second, "new", false); + assertEquals("new", readContent(second)); + } + + @Test + void testGetWriter_appendMode_preservesExistingContent() throws IOException { + StagingManager manager = newManager(); + StagedResource first = new StagedResource(tempDir, "path1", manager); + writeAndClose(first, "hello ", false); + StagedResource second = new StagedResource(tempDir, "path1", manager); + writeAndClose(second, "world", true); + assertEquals("hello world", readContent(second)); + } + + @Test + void testGetWriter_appendMode_onFreshResource_createsFileFromScratch() throws IOException { + StagingManager manager = newManager(); + StagedResource resource = new StagedResource(tempDir, "path1", manager); + writeAndClose(resource, "brand new", true); + assertEquals("brand new", readContent(resource)); + } + + @Test + void testGetWriter_defaultOneArgOverload_behavesSameAsNonAppend() throws IOException { + StagingManager manager = newManager(); + StagedResource first = new StagedResource(tempDir, "path1", manager); + writeAndClose(first, "original content", false); + StagedResource second = new StagedResource(tempDir, "path1", manager); + BufferedWriter writer = second.getWriter(0); + writer.write("new"); + writer.close(); + second.close(); + assertEquals("new", readContent(second)); + } + + @Test + void testDelete_removesFileAndReportsGone() throws IOException { + StagingManager manager = newManager(); + StagedResource resource = new StagedResource(tempDir, "path1", manager); + writeAndClose(resource, "content", false); + assertTrue(resource.isFileResource()); + assertTrue(resource.delete()); + assertTrue(!resource.getFile().exists()); + } + + @Test + void testGetSize_reflectsWrittenContentLength() throws IOException { + StagingManager manager = newManager(); + StagedResource resource = new StagedResource(tempDir, "path1", manager); + writeAndClose(resource, "12345", false); + assertEquals(5, resource.getSize()); + } + + @Test + void testGetState_defaultsToCreateForNewResource() { + StagingManager manager = newManager(); + StagedResource resource = new StagedResource(tempDir, "path1", manager); + assertEquals(State.CREATE, resource.getState()); + } + + private StagingManager newManager() { + return new StagingManager(tempDir.getAbsolutePath(), false); + } + + private void writeAndClose(StagedResource resource, String content, boolean append) throws IOException { + BufferedWriter writer = resource.getWriter(0, append); + writer.write(content); + writer.close(); + resource.close(); + } + + private String readContent(StagedResource resource) throws IOException { + String content = IOUtils.toString(resource.getReader()); + resource.closeReaders(); + return content; + } +} diff --git a/symmetric-server/src/main/java/org/jumpmind/symmetric/web/FileSyncPullUriHandler.java b/symmetric-server/src/main/java/org/jumpmind/symmetric/web/FileSyncPullUriHandler.java index b7c174d439..a875be984c 100644 --- a/symmetric-server/src/main/java/org/jumpmind/symmetric/web/FileSyncPullUriHandler.java +++ b/symmetric-server/src/main/java/org/jumpmind/symmetric/web/FileSyncPullUriHandler.java @@ -21,6 +21,8 @@ package org.jumpmind.symmetric.web; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Base64; import jakarta.servlet.ServletException; import jakarta.servlet.http.HttpServletRequest; @@ -28,6 +30,8 @@ import org.apache.commons.lang3.StringUtils; import org.jumpmind.symmetric.ISymmetricEngine; +import org.jumpmind.symmetric.file.FileSyncPullResult; +import org.jumpmind.symmetric.io.stage.StagedResourceETag; import org.jumpmind.symmetric.model.Node; import org.jumpmind.symmetric.model.ProcessInfo; import org.jumpmind.symmetric.model.ProcessInfo.ProcessStatus; @@ -50,9 +54,12 @@ public void handle(HttpServletRequest req, HttpServletResponse res) throws IOExc ServletUtils.sendError(res, WebConstants.SC_BAD_REQUEST, "Node must be specified"); return; - } else { - log.debug("File sync pull request received from {}", nodeId); } + String batchIdParam = ServletUtils.getParameter(req, WebConstants.BATCH_ID); + String ifETagHeader = req.getHeader(WebConstants.HEADER_IF_ETAG); + String rangeHeader = req.getHeader(WebConstants.HEADER_RANGE); + log.debug("File sync pull request received from {}: batchId={}, {}={}, {}={}", nodeId, batchIdParam, + WebConstants.HEADER_IF_ETAG, ifETagHeader, WebConstants.HEADER_RANGE, rangeHeader); IOutgoingTransport outgoingTransport = createOutgoingTransport(res.getOutputStream(), req.getHeader(WebConstants.HEADER_ACCEPT_CHARSET), engine.getConfigurationService().getSuspendIgnoreChannelLists(nodeId)); @@ -60,15 +67,28 @@ public void handle(HttpServletRequest req, HttpServletResponse res) throws IOExc new ProcessInfoKey(engine.getNodeService().findIdentityNodeId(), nodeId, ProcessType.FILE_SYNC_PULL_HANDLER)); try { - engine.getFileSyncService().sendFiles(processInfo, - engine.getNodeService().findNode(nodeId, true), outgoingTransport); Node targetNode = engine.getNodeService().findNode(nodeId, true); + FileSyncPullResult result = engine.getFileSyncService().prepareFilesForPull(processInfo, targetNode, + batchIdParam, ifETagHeader, rangeHeader); + if (result.getResumeEtag() != null) { + res.setHeader(WebConstants.HEADER_ETAG, quoteEtag(result.getResumeEtag())); + res.setHeader(WebConstants.HEADER_ACCEPT_RANGES, "bytes"); + if (result.isPartialContent()) { + res.setStatus(WebConstants.SC_PARTIAL_CONTENT); + res.setHeader(WebConstants.HEADER_CONTENT_RANGE, + "bytes " + result.getSkipCount() + "-" + (result.getTotalSize() - 1) + "/" + result.getTotalSize()); + } + } + if (result.isEnvelopeFormatUsed()) { + res.setHeader(WebConstants.HEADER_FILESYNC_FORMAT, "1"); + } if (processInfo.getTotalBatchCount() == 0 && targetNode.isVersionGreaterThanOrEqualTo(3, 8, 0)) { ServletUtils.sendError(res, HttpServletResponse.SC_NO_CONTENT, "No files to pull."); } else { res.setContentType("application/zip"); res.addHeader("Content-Disposition", "attachment; filename=\"file-sync.zip\""); + engine.getFileSyncService().writeFilesForPull(processInfo, result, outgoingTransport); } processInfo.setStatus(ProcessStatus.OK); } catch (RuntimeException ex) { @@ -80,4 +100,13 @@ public void handle(HttpServletRequest req, HttpServletResponse res) throws IOExc } } } + + /** + * An entity-tag must be an opaque quoted string (RFC 9110 section 8.8.3). Base64-encoding the JSON first guarantees the payload can never contain a quote + * or other syntax-breaking character; this header is never parsed back by our own client (which tracks its own {@code If-ETag} instead), so the encoding is + * unidirectional. + */ + private static String quoteEtag(StagedResourceETag etag) { + return "\"" + Base64.getEncoder().encodeToString(etag.toJson().getBytes(StandardCharsets.UTF_8)) + "\""; + } } diff --git a/symmetric-server/src/main/java/org/jumpmind/symmetric/web/PullUriHandler.java b/symmetric-server/src/main/java/org/jumpmind/symmetric/web/PullUriHandler.java index 5368502411..f55212d65a 100644 --- a/symmetric-server/src/main/java/org/jumpmind/symmetric/web/PullUriHandler.java +++ b/symmetric-server/src/main/java/org/jumpmind/symmetric/web/PullUriHandler.java @@ -22,10 +22,20 @@ import java.io.IOException; import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Base64; import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; import org.jumpmind.symmetric.common.Constants; +import org.jumpmind.symmetric.common.ParameterConstants; +import org.jumpmind.symmetric.io.stage.IStagedResource; +import org.jumpmind.symmetric.io.stage.IStagedResource.State; +import org.jumpmind.symmetric.io.stage.StagedResourceETag; +import org.jumpmind.symmetric.model.Channel; import org.jumpmind.symmetric.model.NodeChannels; import org.jumpmind.symmetric.model.Node; import org.jumpmind.symmetric.model.NodeSecurity; @@ -52,6 +62,7 @@ * Handles data pulls from other nodes. */ public class PullUriHandler extends AbstractCompressionUriHandler { + private static final Pattern RANGE_PATTERN = Pattern.compile("chars=(\\d{1,18})-"); private INodeService nodeService; private IConfigurationService configurationService; private IDataExtractorService dataExtractorService; @@ -87,51 +98,33 @@ public void handleWithCompression(HttpServletRequest req, HttpServletResponse re nodeChannels.addIgnoreChannels(nodeId, req.getHeader(WebConstants.IGNORED_CHANNELS)); nodeChannels.setChannelQueue(req.getHeader(WebConstants.CHANNEL_QUEUE)); // pull out headers and pass to pull() method - handlePull(nodeId, req.getRemoteHost(), req.getRemoteAddr(), res.getOutputStream(), + String batchIdParam = ServletUtils.getParameter(req, WebConstants.BATCH_ID); + String ifETagHeader = req.getHeader(WebConstants.HEADER_IF_ETAG); + String rangeHeader = req.getHeader(WebConstants.HEADER_RANGE); + log.debug("Pull request from node {} on queue {}: batchId={}, {}={}, {}={}", nodeId, nodeChannels.getChannelQueue(), + batchIdParam, WebConstants.HEADER_IF_ETAG, ifETagHeader, WebConstants.HEADER_RANGE, rangeHeader); + ResumeRequest resumeRequest = new ResumeRequest(nodeId, batchIdParam, ifETagHeader, rangeHeader, nodeChannels.getChannelQueue()); + handlePull(resumeRequest, req.getRemoteHost(), req.getRemoteAddr(), res.getOutputStream(), req.getHeader(WebConstants.HEADER_ACCEPT_CHARSET), res, nodeChannels); log.debug("Pull completed for {} at remote address {}", nodeId, req.getRemoteAddr()); } - protected void handlePull(String nodeId, String remoteHost, String remoteAddress, + protected void handlePull(ResumeRequest resumeRequest, String remoteHost, String remoteAddress, OutputStream outputStream, String encoding, HttpServletResponse res, NodeChannels nodeChannels) throws IOException { + String nodeId = resumeRequest.getNodeId(); NodeSecurity nodeSecurity = nodeService.findNodeSecurity(nodeId, true); long ts = System.currentTimeMillis(); try { NodeChannels remoteSuspendIgnoreChannelsList = configurationService.getSuspendIgnoreChannelLists(); nodeChannels.addSuspendChannels(remoteSuspendIgnoreChannelsList.getSuspendChannels()); nodeChannels.addIgnoreChannels(remoteSuspendIgnoreChannelsList.getIgnoreChannels()); - if (nodeSecurity != null) { - String createdAtNodeId = nodeSecurity.getCreatedAtNodeId(); - if (nodeSecurity.isRegistrationEnabled() && - (createdAtNodeId == null || createdAtNodeId.equals(nodeService.findIdentityNodeId()))) { - registrationService.registerNode(nodeService.findNode(nodeId), remoteHost, - remoteAddress, outputStream, null, null, false); - } else { - IOutgoingTransport outgoingTransport = createOutgoingTransport(outputStream, encoding, - nodeChannels); - ProcessInfo processInfo = statisticManager.newProcessInfo(new ProcessInfoKey( - nodeService.findIdentityNodeId(), nodeChannels.getChannelQueue(), nodeId, ProcessType.PULL_HANDLER_EXTRACT)); - try { - Node targetNode = nodeService.findNode(nodeId, true); - if (Constants.QUEUE_DEFAULT.equals(nodeChannels.getChannelQueue())) { - addReadyQueuesHeader(nodeId, res); - } - List batchList = dataExtractorService.extract(processInfo, targetNode, - nodeChannels.getChannelQueue(), outgoingTransport); - logDataReceivedFromPull(targetNode, batchList, processInfo, remoteHost); - if (processInfo.getStatus() != ProcessStatus.ERROR) { - addPendingBatchCounts(targetNode.getNodeId(), res); - processInfo.setStatus(ProcessStatus.OK); - } - } finally { - if (processInfo.getStatus() != ProcessStatus.OK) { - processInfo.setStatus(ProcessStatus.ERROR); - } - } - outgoingTransport.close(); - } - } else { + if (nodeSecurity == null) { log.warn("Node {} does not exist", nodeId); + } else if (isRegistrationRequired(nodeSecurity)) { + registrationService.registerNode(nodeService.findNode(nodeId), remoteHost, + remoteAddress, outputStream, null, null, false); + } else { + extractAndSendBatches(resumeRequest, remoteHost, outputStream, encoding, res, nodeChannels, nodeId); } } finally { statisticManager.incrementNodesPulled(1); @@ -140,6 +133,111 @@ protected void handlePull(String nodeId, String remoteHost, String remoteAddress log.debug("Pull completed for {} at remote address {} for queue {}", nodeId, remoteAddress, nodeChannels.getChannelQueue()); } + private boolean isRegistrationRequired(NodeSecurity nodeSecurity) { + String createdAtNodeId = nodeSecurity.getCreatedAtNodeId(); + return nodeSecurity.isRegistrationEnabled() + && (createdAtNodeId == null || createdAtNodeId.equals(nodeService.findIdentityNodeId())); + } + + private void extractAndSendBatches(ResumeRequest resumeRequest, String remoteHost, OutputStream outputStream, String encoding, + HttpServletResponse res, NodeChannels nodeChannels, String nodeId) throws IOException { + IOutgoingTransport outgoingTransport = createOutgoingTransport(outputStream, encoding, nodeChannels); + ProcessInfo processInfo = statisticManager.newProcessInfo(new ProcessInfoKey( + nodeService.findIdentityNodeId(), nodeChannels.getChannelQueue(), nodeId, ProcessType.PULL_HANDLER_EXTRACT)); + try { + Node targetNode = nodeService.findNode(nodeId, true); + if (Constants.QUEUE_DEFAULT.equals(nodeChannels.getChannelQueue())) { + addReadyQueuesHeader(nodeId, res); + } + if (StringUtils.isNotBlank(resumeRequest.getBatchIdParam()) && parameterService.is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED) + && handleResume(resumeRequest, outgoingTransport, res, processInfo)) { + processInfo.setStatus(ProcessStatus.OK); + } else { + List batchList = dataExtractorService.extract(processInfo, targetNode, + nodeChannels.getChannelQueue(), outgoingTransport); + logDataReceivedFromPull(targetNode, batchList, processInfo, remoteHost); + if (processInfo.getStatus() != ProcessStatus.ERROR) { + addPendingBatchCounts(targetNode.getNodeId(), res); + processInfo.setStatus(ProcessStatus.OK); + } + } + } finally { + if (processInfo.getStatus() != ProcessStatus.OK) { + processInfo.setStatus(ProcessStatus.ERROR); + } + } + outgoingTransport.close(); + } + + /** + * Serves a single, previously-interrupted batch pull directly from its staged resource, instead of the normal multi-batch {@code extract()} path. Returns + * {@code false} (leaving the response untouched) whenever no valid, fully-staged, file-backed resource exists for the requested batch, so the caller can + * fall through to a normal pull unchanged. + *

+ * {@code request.getQueue()} must match the requested batch's own channel's configured queue - a node can have multiple active pull queues (e.g. "default" + * and "system"), and the client's resume cache is keyed per node, not per queue, so a request for one queue's own batches could otherwise carry a stale or + * mismatched {@code batchId} left over from a different queue's pending resume. + */ + boolean handleResume(ResumeRequest request, IOutgoingTransport outgoingTransport, HttpServletResponse res, ProcessInfo processInfo) { + String nodeId = request.getNodeId(); + long batchId = NumberUtils.toLong(request.getBatchIdParam(), -1L); + if (batchId < 0) { + return false; + } + OutgoingBatch batch = outgoingBatchService.findOutgoingBatch(batchId, nodeId); + IStagedResource stagedResource = dataExtractorService.getStagedResourceForResume(batch); + if (batch == null || stagedResource == null || stagedResource.getState() != State.DONE || !stagedResource.isFileResource()) { + log.debug("Resume requested for batch {} from node {}, but no resumable staged resource was found. Falling back to a full pull.", + batchId, nodeId); + return false; + } + Channel channel = configurationService.getChannel(batch.getChannelId()); + if (channel == null || !channel.getQueue().equals(request.getQueue())) { + log.debug("Resume requested for batch {} from node {} on queue {}, but that batch's channel {} belongs to queue {}. " + + "Falling back to a full pull.", batchId, nodeId, request.getQueue(), batch.getChannelId(), + channel != null ? channel.getQueue() : null); + return false; + } + stagedResource.refreshLastUpdateTime(); + long totalSize = stagedResource.getSize(); + StagedResourceETag etag = new StagedResourceETag(stagedResource.getGenerationTime(), totalSize); + StagedResourceETag requestedETag = StagedResourceETag.fromJson(request.getIfETagHeader()); + Long requestedSkipCount = parseRangeSkipCount(request.getRangeHeader()); + boolean isPartial = etag.equals(requestedETag) && requestedSkipCount != null && requestedSkipCount >= 0 + && requestedSkipCount < totalSize; + long skipCount = isPartial ? requestedSkipCount : 0; + res.setHeader(WebConstants.HEADER_ETAG, quoteEtag(etag)); + res.setHeader(WebConstants.HEADER_ACCEPT_RANGES, "chars"); + if (isPartial) { + res.setStatus(WebConstants.SC_PARTIAL_CONTENT); + res.setHeader(WebConstants.HEADER_CONTENT_RANGE, "chars " + skipCount + "-" + (totalSize - 1) + "/" + totalSize); + } + dataExtractorService.extractSingleBatchForResume(batch, stagedResource, outgoingTransport.getWriter(), skipCount, processInfo); + log.debug("Served {} pull for batch {} to node {} ({} of {} skipped)", isPartial ? "resumed" : "full", batchId, nodeId, + skipCount, totalSize); + return true; + } + + Long parseRangeSkipCount(String rangeHeader) { + if (StringUtils.isBlank(rangeHeader)) { + return null; + } + Matcher matcher = RANGE_PATTERN.matcher(rangeHeader.trim()); + if (!matcher.matches()) { + return null; + } + return Long.parseLong(matcher.group(1)); + } + + /** + * An entity-tag must be an opaque quoted string (RFC 9110 section 8.8.3). Base64-encoding the JSON first guarantees the payload can never contain a quote + * or other syntax-breaking character; this header is never parsed back by our own client (which tracks its own {@code If-ETag} from the body-embedded + * {@code ETAG} line instead), so the encoding is unidirectional. + */ + private static String quoteEtag(StagedResourceETag etag) { + return "\"" + Base64.getEncoder().encodeToString(etag.toJson().getBytes(StandardCharsets.UTF_8)) + "\""; + } + private void addReadyQueuesHeader(String nodeId, HttpServletResponse res) { String headerValue = TransportUtils.buildReadyQueuesHeader(parameterService, configurationService, outgoingBatchService, nodeId); if (headerValue != null) { @@ -162,4 +260,40 @@ private void logDataReceivedFromPull(Node targetNode, List batchL log.info("{} data and {} batches sent during pull request from {}", dataCount, batchesCount, targetNode); } } + + static class ResumeRequest { + private final String nodeId; + private final String batchIdParam; + private final String ifETagHeader; + private final String rangeHeader; + private final String queue; + + ResumeRequest(String nodeId, String batchIdParam, String ifETagHeader, String rangeHeader, String queue) { + this.nodeId = nodeId; + this.batchIdParam = batchIdParam; + this.ifETagHeader = ifETagHeader; + this.rangeHeader = rangeHeader; + this.queue = queue; + } + + String getNodeId() { + return nodeId; + } + + String getBatchIdParam() { + return batchIdParam; + } + + String getIfETagHeader() { + return ifETagHeader; + } + + String getRangeHeader() { + return rangeHeader; + } + + String getQueue() { + return queue; + } + } } diff --git a/symmetric-server/src/test/java/org/jumpmind/symmetric/web/FileSyncPullUriHandlerTest.java b/symmetric-server/src/test/java/org/jumpmind/symmetric/web/FileSyncPullUriHandlerTest.java new file mode 100644 index 0000000000..9e46667c89 --- /dev/null +++ b/symmetric-server/src/test/java/org/jumpmind/symmetric/web/FileSyncPullUriHandlerTest.java @@ -0,0 +1,222 @@ +/** + * Licensed to JumpMind Inc under one or more contributor + * license agreements. See the NOTICE file distributed + * with this work for additional information regarding + * copyright ownership. JumpMind Inc licenses this file + * to you under the GNU Affero General Public License, version 3.0 (AGPLv3) + * (the "License"); you may not use this file except in compliance + * with the License. + * + * You should have received a copy of the GNU Affero General Public License, + * version 3.0 (AGPLv3) along with this library; if not, see + * . + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jumpmind.symmetric.web; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collections; + +import org.jumpmind.symmetric.ISymmetricEngine; +import org.jumpmind.symmetric.file.FileSyncPullResult; +import org.jumpmind.symmetric.io.stage.StagedResourceETag; +import org.jumpmind.symmetric.model.Node; +import org.jumpmind.symmetric.model.NodeChannels; +import org.jumpmind.symmetric.model.OutgoingBatch; +import org.jumpmind.symmetric.model.ProcessInfo; +import org.jumpmind.symmetric.model.ProcessInfoKey; +import org.jumpmind.symmetric.model.ProcessType; +import org.jumpmind.symmetric.service.IConfigurationService; +import org.jumpmind.symmetric.service.IFileSyncService; +import org.jumpmind.symmetric.service.INodeService; +import org.jumpmind.symmetric.service.IParameterService; +import org.jumpmind.symmetric.statistic.IStatisticManager; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import jakarta.servlet.ServletOutputStream; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +class FileSyncPullUriHandlerTest { + private ISymmetricEngine engine; + private IFileSyncService fileSyncService; + private INodeService nodeService; + private FileSyncPullUriHandler handler; + private HttpServletRequest req; + private HttpServletResponse res; + + @BeforeEach + void setUp() throws Exception { + engine = mock(ISymmetricEngine.class); + IParameterService parameterService = mock(IParameterService.class); + when(engine.getParameterService()).thenReturn(parameterService); + fileSyncService = mock(IFileSyncService.class); + when(engine.getFileSyncService()).thenReturn(fileSyncService); + nodeService = mock(INodeService.class); + when(nodeService.findIdentityNodeId()).thenReturn("local"); + when(engine.getNodeService()).thenReturn(nodeService); + IConfigurationService configurationService = mock(IConfigurationService.class); + when(configurationService.getSuspendIgnoreChannelLists(anyString())).thenReturn(new NodeChannels()); + when(engine.getConfigurationService()).thenReturn(configurationService); + IStatisticManager statisticManager = mock(IStatisticManager.class); + when(statisticManager.newProcessInfo(any())).thenAnswer( + inv -> new ProcessInfo(new ProcessInfoKey("local", "node1", ProcessType.FILE_SYNC_PULL_HANDLER))); + when(engine.getStatisticManager()).thenReturn(statisticManager); + handler = new FileSyncPullUriHandler(engine); + req = mock(HttpServletRequest.class); + res = mock(HttpServletResponse.class); + ServletOutputStream outputStream = mock(ServletOutputStream.class); + when(res.getOutputStream()).thenReturn(outputStream); + } + + @Test + void handle_missingNodeId_sendsBadRequestAndSkipsFileSyncService() throws Exception { + when(req.getParameter(WebConstants.NODE_ID)).thenReturn(null); + handler.handle(req, res); + verify(res).sendError(WebConstants.SC_BAD_REQUEST, "Node must be specified"); + verify(fileSyncService, never()).prepareFilesForPull(any(), any(), any(), any(), any()); + } + + @Test + void handle_passesBatchIdAndResumeHeadersThrough() throws Exception { + when(req.getParameter(WebConstants.NODE_ID)).thenReturn("node1"); + when(req.getParameter(WebConstants.BATCH_ID)).thenReturn("42"); + when(req.getHeader(WebConstants.HEADER_IF_ETAG)).thenReturn("{\"etag\":true}"); + when(req.getHeader(WebConstants.HEADER_RANGE)).thenReturn("bytes=10-"); + Node targetNode = new Node(); + targetNode.setNodeId("node1"); + when(nodeService.findNode("node1", true)).thenReturn(targetNode); + ArrayList batches = new ArrayList<>(); + batches.add(new OutgoingBatch()); + when(fileSyncService.prepareFilesForPull(any(), eq(targetNode), eq("42"), eq("{\"etag\":true}"), eq("bytes=10-"))) + .thenAnswer(inv -> { + ProcessInfo processInfo = (ProcessInfo) inv.getArgument(0); + processInfo.incrementBatchCount(); + return FileSyncPullResult.builder().batches(batches).allRequestedBatches(batches).envelopeFormatUsed(false).build(); + }); + handler.handle(req, res); + verify(fileSyncService).prepareFilesForPull(any(), eq(targetNode), eq("42"), eq("{\"etag\":true}"), eq("bytes=10-")); + } + + @Test + void handle_resumeEtagPresentAndPartialContent_setsPartialContentHeaders() throws Exception { + when(req.getParameter(WebConstants.NODE_ID)).thenReturn("node1"); + Node targetNode = new Node(); + targetNode.setNodeId("node1"); + when(nodeService.findNode("node1", true)).thenReturn(targetNode); + StagedResourceETag etag = new StagedResourceETag(100L, 500L); + ArrayList batches = new ArrayList<>(); + batches.add(new OutgoingBatch()); + when(fileSyncService.prepareFilesForPull(any(), eq(targetNode), isNull(), isNull(), isNull())) + .thenAnswer(inv -> { + ProcessInfo processInfo = (ProcessInfo) inv.getArgument(0); + processInfo.incrementBatchCount(); + return FileSyncPullResult.builder().batches(batches).partialContent(true).resumeEtag(etag) + .totalSize(500L).skipCount(200L).build(); + }); + handler.handle(req, res); + verify(res).setHeader(WebConstants.HEADER_ETAG, quotedEtag(etag)); + verify(res).setHeader(WebConstants.HEADER_ACCEPT_RANGES, "bytes"); + verify(res).setStatus(WebConstants.SC_PARTIAL_CONTENT); + verify(res).setHeader(WebConstants.HEADER_CONTENT_RANGE, "bytes 200-499/500"); + } + + @Test + void handle_resumeEtagPresentButFullContent_setsEtagWithoutPartialStatus() throws Exception { + when(req.getParameter(WebConstants.NODE_ID)).thenReturn("node1"); + Node targetNode = new Node(); + targetNode.setNodeId("node1"); + when(nodeService.findNode("node1", true)).thenReturn(targetNode); + StagedResourceETag etag = new StagedResourceETag(100L, 500L); + ArrayList batches = new ArrayList<>(); + batches.add(new OutgoingBatch()); + when(fileSyncService.prepareFilesForPull(any(), eq(targetNode), isNull(), isNull(), isNull())) + .thenAnswer(inv -> { + ProcessInfo processInfo = (ProcessInfo) inv.getArgument(0); + processInfo.incrementBatchCount(); + return FileSyncPullResult.builder().batches(batches).partialContent(false).resumeEtag(etag) + .totalSize(500L).skipCount(0L).build(); + }); + handler.handle(req, res); + verify(res).setHeader(WebConstants.HEADER_ETAG, quotedEtag(etag)); + verify(res).setHeader(WebConstants.HEADER_ACCEPT_RANGES, "bytes"); + verify(res, never()).setStatus(WebConstants.SC_PARTIAL_CONTENT); + verify(res, never()).setHeader(eq(WebConstants.HEADER_CONTENT_RANGE), anyString()); + } + + @Test + void handle_envelopeFormatUsed_setsFileSyncFormatHeader() throws Exception { + when(req.getParameter(WebConstants.NODE_ID)).thenReturn("node1"); + Node targetNode = new Node(); + targetNode.setNodeId("node1"); + when(nodeService.findNode("node1", true)).thenReturn(targetNode); + ArrayList batches = new ArrayList<>(); + batches.add(new OutgoingBatch()); + when(fileSyncService.prepareFilesForPull(any(), eq(targetNode), isNull(), isNull(), isNull())) + .thenAnswer(inv -> { + ProcessInfo processInfo = (ProcessInfo) inv.getArgument(0); + processInfo.incrementBatchCount(); + return FileSyncPullResult.builder().batches(batches).allRequestedBatches(batches).envelopeFormatUsed(true).build(); + }); + handler.handle(req, res); + verify(res).setHeader(WebConstants.HEADER_FILESYNC_FORMAT, "1"); + } + + @Test + void handle_legacyFormat_doesNotSetFileSyncFormatHeaderOrResumeHeaders() throws Exception { + when(req.getParameter(WebConstants.NODE_ID)).thenReturn("node1"); + Node targetNode = new Node(); + targetNode.setNodeId("node1"); + when(nodeService.findNode("node1", true)).thenReturn(targetNode); + ArrayList batches = new ArrayList<>(); + batches.add(new OutgoingBatch()); + when(fileSyncService.prepareFilesForPull(any(), eq(targetNode), isNull(), isNull(), isNull())) + .thenAnswer(inv -> { + ProcessInfo processInfo = (ProcessInfo) inv.getArgument(0); + processInfo.incrementBatchCount(); + return FileSyncPullResult.builder().batches(batches).allRequestedBatches(batches).envelopeFormatUsed(false).build(); + }); + handler.handle(req, res); + verify(res, never()).setHeader(eq(WebConstants.HEADER_FILESYNC_FORMAT), anyString()); + verify(res, never()).setHeader(eq(WebConstants.HEADER_ETAG), anyString()); + verify(res).setContentType("application/zip"); + } + + @Test + void handle_noBatchesForNewPeer_sendsNoContentInsteadOfZipContentType() throws Exception { + when(req.getParameter(WebConstants.NODE_ID)).thenReturn("node1"); + Node targetNode = new Node(); + targetNode.setNodeId("node1"); + targetNode.setSymmetricVersion("3.18.0"); + when(nodeService.findNode("node1", true)).thenReturn(targetNode); + when(fileSyncService.prepareFilesForPull(any(), eq(targetNode), isNull(), isNull(), isNull())) + .thenReturn(FileSyncPullResult.builder().batches(Collections.emptyList()) + .allRequestedBatches(Collections.emptyList()).envelopeFormatUsed(false).build()); + handler.handle(req, res); + verify(res).sendError(HttpServletResponse.SC_NO_CONTENT, "No files to pull."); + verify(res, never()).setContentType("application/zip"); + verify(fileSyncService, never()).writeFilesForPull(any(), any(), any()); + } + + private static String quotedEtag(StagedResourceETag etag) { + return "\"" + Base64.getEncoder().encodeToString(etag.toJson().getBytes(StandardCharsets.UTF_8)) + "\""; + } +} diff --git a/symmetric-server/src/test/java/org/jumpmind/symmetric/web/PullUriHandlerTest.java b/symmetric-server/src/test/java/org/jumpmind/symmetric/web/PullUriHandlerTest.java new file mode 100644 index 0000000000..18aa014514 --- /dev/null +++ b/symmetric-server/src/test/java/org/jumpmind/symmetric/web/PullUriHandlerTest.java @@ -0,0 +1,331 @@ +/** + * Licensed to JumpMind Inc under one or more contributor + * license agreements. See the NOTICE file distributed + * with this work for additional information regarding + * copyright ownership. JumpMind Inc licenses this file + * to you under the GNU Affero General Public License, version 3.0 (AGPLv3) + * (the "License"); you may not use this file except in compliance + * with the License. + * + * You should have received a copy of the GNU Affero General Public License, + * version 3.0 (AGPLv3) along with this library; if not, see + * . + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.jumpmind.symmetric.web; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.BufferedWriter; +import java.io.ByteArrayOutputStream; +import java.io.StringWriter; +import java.util.Collections; + +import org.jumpmind.symmetric.common.Constants; +import org.jumpmind.symmetric.common.ParameterConstants; +import org.jumpmind.symmetric.io.stage.IStagedResource; +import org.jumpmind.symmetric.io.stage.IStagedResource.State; +import org.jumpmind.symmetric.io.stage.StagedResourceETag; +import org.jumpmind.symmetric.model.Channel; +import org.jumpmind.symmetric.model.NodeChannels; +import org.jumpmind.symmetric.model.Node; +import org.jumpmind.symmetric.model.NodeSecurity; +import org.jumpmind.symmetric.model.OutgoingBatch; +import org.jumpmind.symmetric.model.ProcessInfo; +import org.jumpmind.symmetric.model.ProcessInfoKey; +import org.jumpmind.symmetric.model.ProcessType; +import org.jumpmind.symmetric.service.IConfigurationService; +import org.jumpmind.symmetric.service.IDataExtractorService; +import org.jumpmind.symmetric.service.INodeService; +import org.jumpmind.symmetric.service.IOutgoingBatchService; +import org.jumpmind.symmetric.service.IParameterService; +import org.jumpmind.symmetric.service.IRegistrationService; +import org.jumpmind.symmetric.statistic.IStatisticManager; +import org.jumpmind.symmetric.transport.IOutgoingTransport; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import jakarta.servlet.http.HttpServletResponse; + +class PullUriHandlerTest { + private IParameterService parameterService; + private INodeService nodeService; + private IConfigurationService configurationService; + private IDataExtractorService dataExtractorService; + private IRegistrationService registrationService; + private IStatisticManager statisticManager; + private IOutgoingBatchService outgoingBatchService; + private PullUriHandler handler; + private HttpServletResponse res; + private IOutgoingTransport outgoingTransport; + private ProcessInfo processInfo; + + @BeforeEach + void setUp() { + parameterService = mock(IParameterService.class); + nodeService = mock(INodeService.class); + configurationService = mock(IConfigurationService.class); + dataExtractorService = mock(IDataExtractorService.class); + registrationService = mock(IRegistrationService.class); + statisticManager = mock(IStatisticManager.class); + outgoingBatchService = mock(IOutgoingBatchService.class); + handler = new PullUriHandler(parameterService, nodeService, configurationService, dataExtractorService, + registrationService, statisticManager, outgoingBatchService); + res = mock(HttpServletResponse.class); + BufferedWriter writer = new BufferedWriter(new StringWriter()); + outgoingTransport = mock(IOutgoingTransport.class); + when(outgoingTransport.getWriter()).thenReturn(writer); + processInfo = new ProcessInfo(new ProcessInfoKey("me", "node1", ProcessType.PULL_HANDLER_EXTRACT)); + } + + @Test + void handleResume_blankBatchId_returnsFalseWithoutLookup() { + boolean result = handler.handleResume(new PullUriHandler.ResumeRequest("node1", "", null, null, Constants.QUEUE_DEFAULT), + outgoingTransport, res, processInfo); + assertFalse(result); + verify(outgoingBatchService, never()).findOutgoingBatch(anyLong(), anyString()); + } + + @Test + void handleResume_nonNumericBatchId_returnsFalse() { + boolean result = handler.handleResume(new PullUriHandler.ResumeRequest("node1", "not-a-number", null, null, Constants.QUEUE_DEFAULT), + outgoingTransport, res, processInfo); + assertFalse(result); + } + + @Test + void handleResume_missingBatch_returnsFalse() { + when(outgoingBatchService.findOutgoingBatch(1L, "node1")).thenReturn(null); + boolean result = handler.handleResume(new PullUriHandler.ResumeRequest("node1", "1", null, null, Constants.QUEUE_DEFAULT), + outgoingTransport, res, processInfo); + assertFalse(result); + } + + @Test + void handleResume_missingStagedResource_returnsFalse() { + OutgoingBatch batch = new OutgoingBatch(); + batch.setBatchId(1); + when(outgoingBatchService.findOutgoingBatch(1L, "node1")).thenReturn(batch); + when(dataExtractorService.getStagedResourceForResume(batch)).thenReturn(null); + boolean result = handler.handleResume(new PullUriHandler.ResumeRequest("node1", "1", null, null, Constants.QUEUE_DEFAULT), + outgoingTransport, res, processInfo); + assertFalse(result); + } + + @Test + void handleResume_resourceNotDone_returnsFalse() { + OutgoingBatch batch = new OutgoingBatch(); + batch.setBatchId(1); + IStagedResource resource = mock(IStagedResource.class); + when(resource.getState()).thenReturn(State.CREATE); + when(outgoingBatchService.findOutgoingBatch(1L, "node1")).thenReturn(batch); + when(dataExtractorService.getStagedResourceForResume(batch)).thenReturn(resource); + boolean result = handler.handleResume(new PullUriHandler.ResumeRequest("node1", "1", null, null, Constants.QUEUE_DEFAULT), + outgoingTransport, res, processInfo); + assertFalse(result); + } + + @Test + void handleResume_resourceNotFileBacked_returnsFalse() { + OutgoingBatch batch = new OutgoingBatch(); + batch.setBatchId(1); + IStagedResource resource = mock(IStagedResource.class); + when(resource.getState()).thenReturn(State.DONE); + when(resource.isFileResource()).thenReturn(false); + when(outgoingBatchService.findOutgoingBatch(1L, "node1")).thenReturn(batch); + when(dataExtractorService.getStagedResourceForResume(batch)).thenReturn(resource); + boolean result = handler.handleResume(new PullUriHandler.ResumeRequest("node1", "1", null, null, Constants.QUEUE_DEFAULT), + outgoingTransport, res, processInfo); + assertFalse(result); + } + + @Test + void handleResume_channelOnDifferentQueue_returnsFalse() { + OutgoingBatch batch = new OutgoingBatch(); + batch.setBatchId(1); + batch.setChannelId("channel1"); + setUpEligibleResource(batch); + Channel channel = new Channel(); + channel.setQueue(Constants.QUEUE_SYSTEM); + when(configurationService.getChannel("channel1")).thenReturn(channel); + StagedResourceETag etag = new StagedResourceETag(1000L, 500L); + boolean result = handler.handleResume(new PullUriHandler.ResumeRequest("node1", "1", etag.toJson(), "chars=200-", Constants.QUEUE_DEFAULT), + outgoingTransport, res, processInfo); + assertFalse(result); + verify(dataExtractorService, never()).extractSingleBatchForResume(any(), any(), any(), anyLong(), any()); + } + + @Test + void handleResume_channelNotFound_returnsFalse() { + OutgoingBatch batch = new OutgoingBatch(); + batch.setBatchId(1); + batch.setChannelId("channel1"); + setUpEligibleResource(batch); + when(configurationService.getChannel("channel1")).thenReturn(null); + StagedResourceETag etag = new StagedResourceETag(1000L, 500L); + boolean result = handler.handleResume(new PullUriHandler.ResumeRequest("node1", "1", etag.toJson(), "chars=200-", Constants.QUEUE_DEFAULT), + outgoingTransport, res, processInfo); + assertFalse(result); + } + + private IStagedResource setUpEligibleResource(OutgoingBatch batch) { + IStagedResource resource = mock(IStagedResource.class); + when(resource.getState()).thenReturn(State.DONE); + when(resource.isFileResource()).thenReturn(true); + when(resource.getGenerationTime()).thenReturn(1000L); + when(resource.getSize()).thenReturn(500L); + when(outgoingBatchService.findOutgoingBatch(1L, "node1")).thenReturn(batch); + when(dataExtractorService.getStagedResourceForResume(batch)).thenReturn(resource); + Channel channel = new Channel(); + channel.setQueue(Constants.QUEUE_DEFAULT); + when(configurationService.getChannel(batch.getChannelId())).thenReturn(channel); + return resource; + } + + @Test + void handleResume_matchingEtagAndValidRange_returnsPartialContent() { + OutgoingBatch batch = new OutgoingBatch(); + batch.setBatchId(1); + batch.setChannelId("channel1"); + setUpEligibleResource(batch); + StagedResourceETag etag = new StagedResourceETag(1000L, 500L); + boolean result = handler.handleResume(new PullUriHandler.ResumeRequest("node1", "1", etag.toJson(), "chars=200-", Constants.QUEUE_DEFAULT), + outgoingTransport, res, processInfo); + assertTrue(result); + verify(res).setStatus(WebConstants.SC_PARTIAL_CONTENT); + verify(res).setHeader(WebConstants.HEADER_CONTENT_RANGE, "chars 200-499/500"); + verify(res).setHeader(eq(WebConstants.HEADER_ETAG), anyString()); + verify(res).setHeader(WebConstants.HEADER_ACCEPT_RANGES, "chars"); + verify(dataExtractorService).extractSingleBatchForResume(eq(batch), any(), any(), eq(200L), eq(processInfo)); + } + + @Test + void handleResume_staleEtag_fullResendWithNoPartialStatus() { + OutgoingBatch batch = new OutgoingBatch(); + batch.setBatchId(1); + batch.setChannelId("channel1"); + setUpEligibleResource(batch); + StagedResourceETag staleEtag = new StagedResourceETag(999L, 500L); + boolean result = handler.handleResume(new PullUriHandler.ResumeRequest("node1", "1", staleEtag.toJson(), "chars=200-", Constants.QUEUE_DEFAULT), + outgoingTransport, res, processInfo); + assertTrue(result); + verify(res, never()).setStatus(WebConstants.SC_PARTIAL_CONTENT); + verify(res, never()).setHeader(eq(WebConstants.HEADER_CONTENT_RANGE), anyString()); + verify(dataExtractorService).extractSingleBatchForResume(eq(batch), any(), any(), eq(0L), eq(processInfo)); + } + + @Test + void handleResume_missingIfETagHeader_fullResend() { + OutgoingBatch batch = new OutgoingBatch(); + batch.setBatchId(1); + batch.setChannelId("channel1"); + setUpEligibleResource(batch); + boolean result = handler.handleResume(new PullUriHandler.ResumeRequest("node1", "1", null, "chars=200-", Constants.QUEUE_DEFAULT), + outgoingTransport, res, processInfo); + assertTrue(result); + verify(res, never()).setStatus(WebConstants.SC_PARTIAL_CONTENT); + verify(dataExtractorService).extractSingleBatchForResume(eq(batch), any(), any(), eq(0L), eq(processInfo)); + } + + @Test + void handleResume_missingRangeHeader_fullResendEvenWithMatchingEtag() { + OutgoingBatch batch = new OutgoingBatch(); + batch.setBatchId(1); + batch.setChannelId("channel1"); + setUpEligibleResource(batch); + StagedResourceETag etag = new StagedResourceETag(1000L, 500L); + boolean result = handler.handleResume(new PullUriHandler.ResumeRequest("node1", "1", etag.toJson(), null, Constants.QUEUE_DEFAULT), + outgoingTransport, res, processInfo); + assertTrue(result); + verify(res, never()).setStatus(WebConstants.SC_PARTIAL_CONTENT); + verify(dataExtractorService).extractSingleBatchForResume(eq(batch), any(), any(), eq(0L), eq(processInfo)); + } + + @Test + void handleResume_rangeStartAtOrPastTotalSize_fullResend() { + OutgoingBatch batch = new OutgoingBatch(); + batch.setBatchId(1); + batch.setChannelId("channel1"); + setUpEligibleResource(batch); + StagedResourceETag etag = new StagedResourceETag(1000L, 500L); + boolean result = handler.handleResume(new PullUriHandler.ResumeRequest("node1", "1", etag.toJson(), "chars=500-", Constants.QUEUE_DEFAULT), + outgoingTransport, res, processInfo); + assertTrue(result); + verify(res, never()).setStatus(WebConstants.SC_PARTIAL_CONTENT); + verify(dataExtractorService).extractSingleBatchForResume(eq(batch), any(), any(), eq(0L), eq(processInfo)); + } + + @Test + void parseRangeSkipCount_blankHeader_returnsNull() { + assertNull(handler.parseRangeSkipCount(null)); + assertNull(handler.parseRangeSkipCount("")); + assertNull(handler.parseRangeSkipCount(" ")); + } + + @Test + void parseRangeSkipCount_malformedHeader_returnsNull() { + assertNull(handler.parseRangeSkipCount("chars=abc-")); + assertNull(handler.parseRangeSkipCount("not-a-range-header")); + assertNull(handler.parseRangeSkipCount("chars=100-200")); + assertNull(handler.parseRangeSkipCount("chars=123456789012345678901-")); + } + + @Test + void parseRangeSkipCount_validHeader_returnsSkipCount() { + assertEquals(1234L, handler.parseRangeSkipCount("chars=1234-")); + } + + @Test + void parseRangeSkipCount_headerWithWhitespace_isTrimmed() { + assertEquals(42L, handler.parseRangeSkipCount(" chars=42- ")); + } + + @Test + void handlePull_resumeEligibleAndEnabled_skipsNormalExtract() throws Exception { + PullUriHandler spyHandler = spy(handler); + NodeSecurity nodeSecurity = new NodeSecurity(); + when(nodeService.findNodeSecurity("node1", true)).thenReturn(nodeSecurity); + when(configurationService.getSuspendIgnoreChannelLists()).thenReturn(new NodeChannels()); + when(nodeService.findNode("node1", true)).thenReturn(new Node()); + when(statisticManager.newProcessInfo(any())).thenReturn(processInfo); + when(parameterService.is(ParameterConstants.TRANSPORT_HTTP_RESUME_ENABLED)).thenReturn(true); + doReturn(true).when(spyHandler).handleResume(any(), any(), any(), any()); + spyHandler.handlePull(new PullUriHandler.ResumeRequest("node1", "1", null, null, Constants.QUEUE_DEFAULT), "host", "1.2.3.4", + new ByteArrayOutputStream(), null, res, new NodeChannels()); + verify(dataExtractorService, never()).extract(any(), any(), any(), any()); + } + + @Test + void handlePull_noBatchId_neverInvokesResume() throws Exception { + PullUriHandler spyHandler = spy(handler); + NodeSecurity nodeSecurity = new NodeSecurity(); + when(nodeService.findNodeSecurity("node1", true)).thenReturn(nodeSecurity); + when(configurationService.getSuspendIgnoreChannelLists()).thenReturn(new NodeChannels()); + when(nodeService.findNode("node1", true)).thenReturn(new Node()); + when(statisticManager.newProcessInfo(any())).thenReturn(processInfo); + when(dataExtractorService.extract(any(), any(), any(), any())).thenReturn(Collections.emptyList()); + spyHandler.handlePull(new PullUriHandler.ResumeRequest("node1", null, null, null, Constants.QUEUE_DEFAULT), "host", "1.2.3.4", + new ByteArrayOutputStream(), null, res, new NodeChannels()); + verify(spyHandler, never()).handleResume(any(), any(), any(), any()); + verify(dataExtractorService).extract(any(), any(), any(), any()); + } +}