Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 51 additions & 2 deletions src/core/cloud_filesystem.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -445,8 +445,57 @@ void CloudFileSystem::RemoveFile(const string& path, optional_ptr<FileOpener>) {
cache_.InvalidateItem(b.Scheme(), path);
}

void CloudFileSystem::MoveFile(const string&, const string&, optional_ptr<FileOpener>) {
Throw("cloudfs: MoveFile not yet implemented");
void CloudFileSystem::MoveFile(const string& src_url, const string& dst_url,
optional_ptr<FileOpener>) {
auto& b = BackendFor(src_url);
if (!b.CanHandle(dst_url))
Throw("cloudfs: move across providers is not supported");

std::string src_raw, src_path, dst_raw, dst_path, err;
Throw(!b.ParseUrl(src_url, src_raw, src_path, err) ? err : "");
Throw(!b.ParseUrl(dst_url, dst_raw, dst_path, err) ? err : "");
if (src_raw != dst_raw)
Throw("cloudfs: cross-root move not supported; source and destination must share the same "
"storage root");

// Resolve root (with cache)
std::string root_id;
if (!cache_.GetRootId(b.Scheme(), src_raw, root_id)) {
std::string tok;
Throw(!GetToken(b.Scheme(), tok, err) ? err : "");
Throw(!b.ResolveRoot(src_raw, tok, root_id, err) ? err : "");
cache_.PutRootId(b.Scheme(), src_raw, root_id);
}
std::string tok;
Throw(!GetToken(b.Scheme(), tok, err) ? err : "");

// Stat the source item
CloudItem src_item;
Throw(!b.Stat(root_id, src_path, tok, src_item, err) ? "source not found: " + err : "");

// Decompose destination into parent path + filename
auto last_slash = dst_path.rfind('/');
std::string dst_name =
(last_slash == std::string::npos) ? dst_path : dst_path.substr(last_slash + 1);
Throw(dst_name.empty() ? "cloudfs: destination must include a filename" : "");
std::string dst_parent_path =
(last_slash == std::string::npos || last_slash == 0) ? "/" : dst_path.substr(0, last_slash);

if (dst_name.empty())
Throw("cloudfs: destination path must include a filename, not a directory path");

// Stat the destination parent folder
CloudItem dst_parent;
Throw(!b.Stat(root_id, dst_parent_path, tok, dst_parent, err)
? "destination parent not found: " + err
: "");

// Perform server-side move
Throw(!b.MoveItem(root_id, src_item, dst_parent, dst_name, tok, err) ? err : "");

// Invalidate affected cache entries
cache_.InvalidateItem(b.Scheme(), src_url);
cache_.InvalidatePrefix(b.Scheme(), b.Scheme() + "://" + src_raw + dst_parent_path);
}

bool CloudFileSystem::ListFiles(const string& dir,
Expand Down
8 changes: 8 additions & 0 deletions src/include/core/cloud_backend.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,14 @@ class ICloudBackend {
return false;
}

// Optional server-side move/rename (avoids delete+re-upload for large files).
virtual bool MoveItem(const std::string& root, const CloudItem& src_item,
const CloudItem& dst_parent, const std::string& dst_name,
const std::string& access_token, std::string& err) {
err = Name() + ": server-side move not supported";
return false;
}

// ── Root resolution ───────────────────────────────────────────────────────

// Resolve the root string (parsed by ParseUrl) to an internal drive/bucket ID.
Expand Down
3 changes: 3 additions & 0 deletions src/include/providers/dropbox_backend.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,9 @@ class DropboxBackend : public ICloudBackend {
const std::string& dst_parent_id, const std::string& dst_name,
const std::string& token, std::string& err) override;

bool MoveItem(const std::string& root, const CloudItem& src_item, const CloudItem& dst_parent,
const std::string& dst_name, const std::string& token, std::string& err) override;

private:
CloudItem ParseMetadata(const std::string& json) const;

Expand Down
3 changes: 3 additions & 0 deletions src/include/providers/gdrive_backend.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,9 @@ class GDriveBackend : public ICloudBackend {
const std::string& dst_parent_id, const std::string& dst_name,
const std::string& token, std::string& err) override;

bool MoveItem(const std::string& root, const CloudItem& src_item, const CloudItem& dst_parent,
const std::string& dst_name, const std::string& token, std::string& err) override;

bool AbortUpload(const CloudUploadSession& session, const std::string& token,
std::string& err) override;

Expand Down
7 changes: 7 additions & 0 deletions src/include/providers/onedrive_backend.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,13 @@ class OneDriveBackend : public ICloudBackend {
const std::string& name, const std::string& token, CloudItem& out,
std::string& err) override;

bool CopyItem(const std::string& root, const std::string& src_id,
const std::string& dst_parent_id, const std::string& dst_name,
const std::string& token, std::string& err) override;

bool MoveItem(const std::string& root, const CloudItem& src_item, const CloudItem& dst_parent,
const std::string& dst_name, const std::string& token, std::string& err) override;

// Microsoft Graph has no recursive-list API; override with BFS for explicit control.
bool ListFolderRecursive(const std::string& root, const std::string& folder_id,
const std::string& token,
Expand Down
4 changes: 3 additions & 1 deletion src/include/providers/sftp_backend.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,9 @@ class SFTPBackend : public ICloudBackend {
const std::string& name, const std::string& token, CloudItem& out,
std::string& err) override;

bool MoveItem(const std::string& root, const CloudItem& src_item, const CloudItem& dst_parent,
const std::string& dst_name, const std::string& token, std::string& err) override;

// SFTP protocol has no server-side copy; implement as client-side read+write.
bool CopyItem(const std::string& root, const std::string& src_id,
const std::string& dst_parent_id, const std::string& dst_name,
Expand All @@ -110,7 +113,6 @@ class SFTPBackend : public ICloudBackend {
const std::string& token,
const std::function<void(const CloudItem&)>& cb,
std::string& err) override;

bool AbortUpload(const CloudUploadSession& session, const std::string& token,
std::string& err) override;

Expand Down
3 changes: 3 additions & 0 deletions src/include/providers/sharepoint_backend.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,9 @@ class SharePointBackend : public ICloudBackend {
const std::string& dst_parent_id, const std::string& dst_name,
const std::string& token, std::string& err) override;

bool MoveItem(const std::string& root, const CloudItem& src_item, const CloudItem& dst_parent,
const std::string& dst_name, const std::string& token, std::string& err) override;

// Microsoft Graph has no recursive-list API; override with BFS for explicit control.
bool ListFolderRecursive(const std::string& root, const std::string& folder_id,
const std::string& token,
Expand Down
3 changes: 3 additions & 0 deletions src/include/providers/vfs_backend.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,9 @@ class VFSBackend : public ICloudBackend {
const std::string& name, const std::string& token, CloudItem& out,
std::string& err) override;

bool MoveItem(const std::string& root, const CloudItem& src_item, const CloudItem& dst_parent,
const std::string& dst_name, const std::string& token, std::string& err) override;

// VFS agent supports ?recursive=true for a single deep-listing HTTP call.
bool ListFolderRecursive(const std::string& root, const std::string& folder_id,
const std::string& token,
Expand Down
20 changes: 20 additions & 0 deletions src/providers/dropbox/dropbox_backend.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -278,4 +278,24 @@ bool DropboxBackend::CopyItem(const std::string&, const std::string& src_id,
return true;
}

bool DropboxBackend::MoveItem(const std::string&, const CloudItem& src_item,
const CloudItem& dst_parent, const std::string& dst_name,
const std::string& tok, std::string& err) {
// Dropbox move_v2: from_path accepts opaque Dropbox ID ("id:...") but to_path must be a path.
HttpRequest req;
req.method = "POST";
req.SetBearerAuth(tok);
req.headers["Content-Type"] = "application/json";
req.url = std::string(kApiBase) + "/files/move_v2";
req.body = "{\"from_path\":\"" + JsonUtil::EscapeJsonString(src_item.path) +
"\",\"to_path\":\"" + JsonUtil::EscapeJsonString(dst_parent.path + "/" + dst_name) +
"\"}";
auto resp = http_.Execute(req);
if (!resp.ok()) {
err = "move failed " + std::to_string(resp.status);
return false;
}
return true;
}

} // namespace duckdb
51 changes: 49 additions & 2 deletions src/providers/gdrive/gdrive_backend.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,54 @@ bool GDriveBackend::CopyItem(const std::string&, const std::string& src_id,
return true;
}

bool GDriveBackend::MoveItem(const std::string&, const CloudItem& src_item,
const CloudItem& dst_parent, const std::string& dst_name,
const std::string& tok, std::string& err) {
// Fetch current parents so we can remove ALL of them (avoids multi-parent artifacts).
std::string meta_url =
kApiBase + "/files/" + src_item.id + "?fields=parents&supportsAllDrives=true";
auto meta_resp = http_.Get(meta_url, tok);
if (!meta_resp.ok()) {
err = "move: failed to fetch file metadata (" + std::to_string(meta_resp.status) + ")";
return false;
}
// Safely extract all parent IDs from JSON array ["id1","id2",...]
std::string remove_parents;
{
const auto& body = meta_resp.body;
auto p_pos = body.find("\"parents\"");
auto arr_start = (p_pos != std::string::npos) ? body.find('[', p_pos) : std::string::npos;
auto arr_end =
(arr_start != std::string::npos) ? body.find(']', arr_start) : std::string::npos;
for (size_t i = (arr_start != std::string::npos ? arr_start + 1 : body.size());
arr_end != std::string::npos && i < arr_end;) {
auto q1 = body.find('"', i);
if (q1 == std::string::npos || q1 >= arr_end)
break;
auto q2 = body.find('"', q1 + 1);
if (q2 == std::string::npos || q2 >= arr_end)
break;
if (!remove_parents.empty())
remove_parents += ",";
remove_parents += body.substr(q1 + 1, q2 - q1 - 1);
i = q2 + 1;
}
}

std::string url = kApiBase + "/files/" + src_item.id + "?supportsAllDrives=true";
if (!remove_parents.empty())
url += "&removeParents=" + remove_parents; // GDrive IDs are safe alphanumeric
url += "&addParents=" + dst_parent.id;

std::string body = "{\"name\":\"" + JsonUtil::EscapeJsonString(dst_name) + "\"}";
auto resp = http_.Patch(url, tok, body);
if (!resp.ok()) {
err = "move failed (" + std::to_string(resp.status) + ")";
return false;
}
return true;
}

bool GDriveBackend::AbortUpload(const CloudUploadSession& session, const std::string& tok,
std::string& err) {
if (session.upload_url.empty())
Expand Down Expand Up @@ -251,7 +299,7 @@ bool GDriveBackend::ListFolderRecursive(const std::string&, const std::string& f

auto resp = http_.Get(url, tok);
if (!resp.ok()) {
err = "list recursive failed " + std::to_string(resp.status);
err = "list recursive failed (" + std::to_string(resp.status) + ")";
return false;
}
for (auto& j : JsonUtil::GetArray(resp.body, "files")) {
Expand All @@ -268,5 +316,4 @@ bool GDriveBackend::ListFolderRecursive(const std::string&, const std::string& f
} while (!page_token.empty());
return true;
}

} // namespace duckdb
31 changes: 31 additions & 0 deletions src/providers/onedrive/onedrive_backend.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,22 @@ bool OneDriveBackend::CreateFolder(const std::string& root, const std::string& p
return true;
}

bool OneDriveBackend::CopyItem(const std::string& root, const std::string& src_id,
const std::string& dst_parent_id, const std::string& dst_name,
const std::string& tok, std::string& err) {
std::string url =
"https://graph.microsoft.com/v1.0/drives/" + root + "/items/" + src_id + "/copy";
std::string body = "{\"parentReference\":{\"id\":\"" +
JsonUtil::EscapeJsonString(dst_parent_id) + "\"},\"name\":\"" +
JsonUtil::EscapeJsonString(dst_name) + "\"}";
auto resp = http_.Post(url, tok, body);
if (resp.status != 202) {
err = "copy failed (" + std::to_string(resp.status) + ")";
return false;
}
return true;
}

bool OneDriveBackend::ListFolderRecursive(const std::string& root, const std::string& folder_id,
const std::string& tok,
const std::function<void(const CloudItem&)>& cb,
Expand All @@ -187,6 +203,21 @@ bool OneDriveBackend::ListFolderRecursive(const std::string& root, const std::st
return true;
}

bool OneDriveBackend::MoveItem(const std::string& root, const CloudItem& src_item,
const CloudItem& dst_parent, const std::string& dst_name,
const std::string& tok, std::string& err) {
std::string url = "https://graph.microsoft.com/v1.0/drives/" + root + "/items/" + src_item.id;
std::string body = "{\"name\":\"" + JsonUtil::EscapeJsonString(dst_name) +
"\",\"parentReference\":{\"id\":\"" +
JsonUtil::EscapeJsonString(dst_parent.id) + "\"}}";
auto resp = http_.Patch(url, tok, body);
if (!resp.ok()) {
err = "move failed (" + std::to_string(resp.status) + ")";
return false;
}
return true;
}

bool OneDriveBackend::AbortUpload(const CloudUploadSession& session, const std::string& tok,
std::string& err) {
if (session.upload_url.empty())
Expand Down
19 changes: 19 additions & 0 deletions src/providers/sftp/sftp_backend.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -397,6 +397,25 @@ bool SFTPBackend::CreateFolder(const std::string& root, const std::string& paren
return true;
}

bool SFTPBackend::MoveItem(const std::string& root, const CloudItem& src_item,
const CloudItem& dst_parent, const std::string& dst_name,
const std::string& token, std::string& err) {
auto& conn = GetConnection(root, token, err);
if (!conn.IsValid())
return false;
std::string dst_path = dst_parent.id + "/" + dst_name;
// LIBSSH2_SFTP_RENAME_OVERWRITE allows replacing an existing destination
int rc = libssh2_sftp_rename_ex(
conn.sftp, src_item.id.c_str(), (unsigned int)src_item.id.size(), dst_path.c_str(),
(unsigned int)dst_path.size(),
LIBSSH2_SFTP_RENAME_OVERWRITE | LIBSSH2_SFTP_RENAME_ATOMIC | LIBSSH2_SFTP_RENAME_NATIVE);
if (rc != 0) {
err = "sftp rename failed: " + src_item.id + " -> " + dst_path;
return false;
}
return true;
}

bool SFTPBackend::AbortUpload(const CloudUploadSession& session, const std::string& token,
std::string& err) {
// upload_url contains the remote file path; item_id is used as root (user@host:port)
Expand Down
15 changes: 15 additions & 0 deletions src/providers/sharepoint/sharepoint_backend.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,21 @@ bool SharePointBackend::CopyItem(const std::string& root, const std::string& src
return true;
}

bool SharePointBackend::MoveItem(const std::string& root, const CloudItem& src_item,
const CloudItem& dst_parent, const std::string& dst_name,
const std::string& tok, std::string& err) {
std::string url = "https://graph.microsoft.com/v1.0/drives/" + root + "/items/" + src_item.id;
std::string body = "{\"name\":\"" + JsonUtil::EscapeJsonString(dst_name) +
"\",\"parentReference\":{\"id\":\"" +
JsonUtil::EscapeJsonString(dst_parent.id) + "\"}}";
auto resp = http_.Patch(url, tok, body);
if (!resp.ok()) {
err = "move failed (" + std::to_string(resp.status) + ")";
return false;
}
return true;
}

bool SharePointBackend::ListFolderRecursive(const std::string& root, const std::string& folder_id,
const std::string& tok,
const std::function<void(const CloudItem&)>& cb,
Expand Down
13 changes: 13 additions & 0 deletions src/providers/vfs/vfs_backend.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,19 @@ bool VFSBackend::CreateFolder(const std::string& root, const std::string& parent
return true;
}

bool VFSBackend::MoveItem(const std::string& root, const CloudItem& src_item,
const CloudItem& dst_parent, const std::string& dst_name,
const std::string& token, std::string& err) {
std::string dst_path = dst_parent.id + "/" + dst_name;
std::string body = JsonUtil::MakeObject({{"from", src_item.id}, {"to", dst_path}});
auto resp = http_.Post(root + "/v1/move", token, body);
if (!resp.ok()) {
err = "move failed (" + std::to_string(resp.status) + ")";
return false;
}
return true;
}

bool VFSBackend::ListFolderRecursive(const std::string& root, const std::string& folder_id,
const std::string& token,
const std::function<void(const CloudItem&)>& cb,
Expand Down
Empty file modified test/test_suite.py
100755 → 100644
Empty file.
Loading