issue: 文件存储支持S3对象存储 #24

README.md

@@ -233,6 +233,16 @@ PODCAST_VOICE=alloy    # Options: alloy, echo, fable, onyx, nova, shimmer
 # Feature Flags
 ALLOW_DELETE=true
 ALLOW_MULTIPLE_NOTES_OF_SAME_TYPE=true
+
+# S3 / Ceph Storage
+S3_ENDPOINT=https://s3.example.com
+S3_REGION=us-east-1   # default us-east-1
+S3_ACCESS_KEY=yourkey
+S3_SECRET_KEY=yoursecret
+S3_BUCKET=notex-uploads
+S3_FORCE_PATH_STYLE=true  # usually true for Ceph/MinIO
+S3_SKIP_TLS_VERIFY=true # default false
+
 ```
 
 ## 🔧 Development

README_CN.md

@@ -233,6 +233,15 @@ PODCAST_VOICE=alloy    # 选项：alloy、echo、fable、onyx、nova、shimmer
 # 功能开关
 ALLOW_DELETE=true
 ALLOW_MULTIPLE_NOTES_OF_SAME_TYPE=true
+
+# S3 / Ceph 对象存储，文件上传
+S3_ENDPOINT=https://s3.example.com
+S3_REGION=us-east-1   # default `us-east-1`
+S3_ACCESS_KEY=yourkey
+S3_SECRET_KEY=yoursecret
+S3_BUCKET=notex-uploads
+S3_FORCE_PATH_STYLE=true  # usually true for Ceph/MinIO
+S3_SKIP_TLS_VERIFY=true # default false
 ```
 
 ## 🔧 开发

README_zh-tw.md

@@ -233,6 +233,15 @@ PODCAST_VOICE=alloy    # 選項：alloy、echo、fable、onyx、nova、shimmer
 # 功能標誌
 ALLOW_DELETE=true
 ALLOW_MULTIPLE_NOTES_OF_SAME_TYPE=true
+
+# S3 / Ceph 對象儲存，檔案上傳
+S3_ENDPOINT=https://s3.example.com
+S3_REGION=us-east-1   # default `us-east-1`
+S3_ACCESS_KEY=yourkey
+S3_SECRET_KEY=yoursecret
+S3_BUCKET=notex-uploads
+S3_FORCE_PATH_STYLE=true  # usually true for Ceph/MinIO
+S3_SKIP_TLS_VERIFY=true # default false
 ```
 
 ## 🔧 開發

backend/config.go

@@ -11,8 +11,8 @@ import (
 // Config holds the application configuration
 type Config struct {
 	// Server settings
-	ServerHost string
-	ServerPort string
+	ServerHost    string
+	ServerPort    string
 	MaxUploadSize int64 // Maximum upload file size in bytes (default: 100MB)
 
 	// LLM settings
@@ -79,11 +79,20 @@ type Config struct {
 	GoogleRedirectURL  string
 
 	// Test Mode
-	EnableTestMode     bool
-	TestUserID         string
-	TestUserName       string
-	TestUserEmail      string
-	TestUserAvatar     string
+	EnableTestMode bool
+	TestUserID     string
+	TestUserName   string
+	TestUserEmail  string
+	TestUserAvatar string
+
+	// Optional S3 / Ceph storage configuration
+	S3Endpoint       string
+	S3Region         string // region string, may be blank for Ceph
+	S3AccessKey      string
+	S3SecretKey      string
+	S3Bucket         string
+	S3ForcePathStyle bool
+	S3SkipTLSVerify  bool
 }
 
 // loadEnv loads .env file if it exists (ignoring errors if file not found)
@@ -147,11 +156,20 @@ func LoadConfig() Config {
 		GoogleClientSecret: getEnv("GOOGLE_CLIENT_SECRET", ""),
 		GoogleRedirectURL:  getEnv("GOOGLE_REDIRECT_URL", ""),
 
-		EnableTestMode:     getEnvBool("ENABLE_TEST_MODE", false),
-		TestUserID:         getEnv("TEST_USER_ID", "test-user-123"),
-		TestUserName:       getEnv("TEST_USER_NAME", "测试用户"),
-		TestUserEmail:      getEnv("TEST_USER_EMAIL", "test@example.com"),
-		TestUserAvatar:     getEnv("TEST_USER_AVATAR", ""),
+		EnableTestMode: getEnvBool("ENABLE_TEST_MODE", false),
+		TestUserID:     getEnv("TEST_USER_ID", "test-user-123"),
+		TestUserName:   getEnv("TEST_USER_NAME", "测试用户"),
+		TestUserEmail:  getEnv("TEST_USER_EMAIL", "test@example.com"),
+		TestUserAvatar: getEnv("TEST_USER_AVATAR", ""),
+
+		// S3 / Ceph storage
+		S3Endpoint:       getEnv("S3_ENDPOINT", ""),
+		S3Region:         getEnv("S3_REGION", "us-east-1"),
+		S3AccessKey:      getEnv("S3_ACCESS_KEY", ""),
+		S3SecretKey:      getEnv("S3_SECRET_KEY", ""),
+		S3Bucket:         getEnv("S3_BUCKET", ""),
+		S3ForcePathStyle: getEnvBool("S3_FORCE_PATH_STYLE", true),
+		S3SkipTLSVerify:  getEnvBool("S3_SKIP_TLS_VERIFY", false),
 	}
 
 	// Auto-detect provider from base URL or model name

backend/server.go

@@ -2,9 +2,11 @@ package backend
 
 import (
 	"context"
+	"crypto/tls"
 	"database/sql"
 	"embed"
 	"fmt"
+	"io"
 	"io/fs"
 	"net/http"
 	"net/url"
@@ -14,6 +16,14 @@ import (
 	"sync"
 	"time"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
+
+	// "github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+
 	"github.com/gin-gonic/gin"
 	"github.com/google/uuid"
 	"github.com/kataras/golog"
@@ -30,10 +40,11 @@ type Server struct {
 	agent       *Agent
 	http        *gin.Engine
 	auth        *AuthHandler
+	s3Client    *s3.Client
 	// Track which notebooks have been loaded into vector store
 	loadedNotebooks map[string]bool
 	vectorMutex     sync.RWMutex
-	memoryManager *MemoryManager
+	memoryManager   *MemoryManager
 }
 
 // NewServer creates a new server
@@ -86,13 +97,20 @@ func NewServer(cfg Config) (*Server, error) {
 	// Set max upload size for multipart forms
 	router.MaxMultipartMemory = cfg.MaxUploadSize
 
+	// Initialize S3 client if configuration present
+	s3Client, err := NewS3Client(cfg)
+	if err != nil {
+		return nil, fmt.Errorf("failed to initialize s3 client: %w", err)
+	}
+
 	s := &Server{
 		cfg:             cfg,
 		vectorStore:     vectorStore,
 		store:           store,
 		agent:           agent,
 		http:            router,
 		auth:            authHandler,
+		s3Client:        s3Client,
 		loadedNotebooks: make(map[string]bool),
 		memoryManager:   memoryManager,
 	}
@@ -105,6 +123,41 @@ func NewServer(cfg Config) (*Server, error) {
 	return s, nil
 }
 
+// NewS3Client create a new S3 client if S3 configuration is provided, otherwise returns nil
+func NewS3Client(cfg Config) (*s3.Client, error) {
+	var s3Client *s3.Client
+	if cfg.S3Endpoint != "" {
+		// sanitize endpoint and ensure region
+		cfg.S3Endpoint = strings.TrimRight(cfg.S3Endpoint, "/")
+		ctxCfg := context.Background()
+		// Prepare HTTP client with optional TLS skip verify for self-signed certs
+		httpClient := http.DefaultClient
+		if cfg.S3SkipTLSVerify {
+			transport := &http.Transport{
+				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+			}
+			httpClient = &http.Client{Transport: transport}
+			golog.Warnf("S3_SKIP_TLS_VERIFY is enabled: TLS certificate verification will be skipped for S3 endpoint %s", cfg.S3Endpoint)
+		}
+
+		awsCfg, err := config.LoadDefaultConfig(ctxCfg,
+			config.WithBaseEndpoint(cfg.S3Endpoint),
+			config.WithRegion(cfg.S3Region),
+			config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(cfg.S3AccessKey, cfg.S3SecretKey, "")),
+			config.WithHTTPClient(httpClient),
+			config.WithRequestChecksumCalculation(aws.RequestChecksumCalculationWhenRequired),
+		)
+		if err != nil {
+			return nil, fmt.Errorf("failed to configure s3 client: %w", err)
+		}
+		s3Client = s3.NewFromConfig(awsCfg, func(o *s3.Options) {
+			o.UsePathStyle = cfg.S3ForcePathStyle
+		})
+		golog.Infof("✅ s3 client initialized (bucket=%s endpoint=%s)", cfg.S3Bucket, cfg.S3Endpoint)
+	}
+	return s3Client, nil
+}
+
 // setupRoutes configures all routes
 func (s *Server) setupRoutes() {
 	// Serve static files from embedded filesystem (no audit)
@@ -615,6 +668,30 @@ func (s *Server) handleDeleteSource(c *gin.Context) {
 		return
 	}
 
+	// try delete file (safe type assertion and non-existence handling)
+	if v, ok := source.Metadata["path"]; ok {
+		pathStr := v.(string)
+		if err := os.Remove(pathStr); err != nil {
+			golog.Errorf("failed to delete file: %s", pathStr)
+		}
+	}
+
+	// if s3Client init delete object (safe type assertion)
+	if s.s3Client != nil {
+		if v, ok := source.Metadata["s3_key"]; ok {
+			s3Key := v.(string)
+			_, err := s.s3Client.DeleteObject(ctx, &s3.DeleteObjectInput{
+				Bucket: aws.String(s.cfg.S3Bucket),
+				Key:    aws.String(s3Key),
+			})
+			if err != nil {
+				golog.Errorf("failed to delete S3 object: %v", err)
+			}
+		} else {
+			golog.Errorf("source not exist s3_key: %s", sourceID)
+		}
+	}
+
 	if err := s.store.DeleteSource(ctx, sourceID); err != nil {
 		c.JSON(http.StatusInternalServerError, ErrorResponse{Error: "Failed to delete source"})
 		return
@@ -678,16 +755,6 @@ func (s *Server) handleUpload(c *gin.Context) {
 		return
 	}
 
-	// Create source
-	source := &Source{
-		NotebookID: notebookID,
-		Name:       file.Filename, // Keep original filename for display
-		Type:       "file",
-		FileName:   uniqueFileName, // Store unique filename
-		FileSize:   file.Size,
-		Metadata:   map[string]interface{}{"path": tempPath, "user_id": userID},
-	}
-
 	// Extract content
 	content, err := s.vectorStore.ExtractDocument(ctx, tempPath)
 	if err != nil {
@@ -697,7 +764,33 @@ func (s *Server) handleUpload(c *gin.Context) {
 		c.JSON(http.StatusInternalServerError, ErrorResponse{Error: fmt.Sprintf("Failed to extract document content: %v", err)})
 		return
 	}
-	source.Content = content
+
+	source := &Source{
+		NotebookID: notebookID,
+		Name:       file.Filename, // Keep original filename for display
+		Type:       "file",
+		FileName:   uniqueFileName, // Store unique filename
+		FileSize:   file.Size,
+		Content:    content,
+		Metadata:   map[string]interface{}{"user_id": userID},
+	}
+
+	// If S3 is configured, upload then cleanup local copy and record key
+	if s.s3Client != nil {
+		s3Key := fmt.Sprintf("%s/%s", userID, uniqueFileName)
+		if err := s.uploadToS3(ctx, tempPath, s3Key); err != nil {
+			golog.Errorf("failed to upload to S3: %v", err)
+			// remove local copy
+			os.Remove(tempPath)
+			c.JSON(http.StatusInternalServerError, ErrorResponse{Error: "Failed to upload file to storage"})
+			return
+		}
+		source.Metadata["s3_key"] = s3Key
+		// local copy no longer needed once stored remotely
+		os.Remove(tempPath)
+	} else {
+		source.Metadata["path"] = tempPath
+	}
 
 	if err := s.store.CreateSource(ctx, source); err != nil {
 		golog.Errorf("failed to create source: %v", err)
@@ -1445,6 +1538,70 @@ func (s *Server) handleServeFile(c *gin.Context) {
 		filename, notebookID, isPublic, userID)
 }
 
+// uploadToS3 uploads a local file to the configured S3 bucket using the
+// provided object key. The caller is responsible for creating and closing the
+// local file. This helper returns any error from the SDK directly.
+func (s *Server) uploadToS3(ctx context.Context, localPath, key string) error {
+	if s.s3Client == nil {
+		return fmt.Errorf("s3 client not configured")
+	}
+	f, err := os.Open(localPath)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	uploader := manager.NewUploader(s.s3Client)
+	_, err = uploader.Upload(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(s.cfg.S3Bucket),
+		Key:    aws.String(key),
+		Body:   f,
+	})
+
+	// ERROR: XAmzContentSHA256Mismatch, like s3Client config at line 124
+	// trans := transfermanager.New(s.s3Client)
+	// _, err = trans.UploadObject(ctx,
+	// 	&transfermanager.UploadObjectInput{
+	// 		Bucket: aws.String(s.cfg.S3Bucket),
+	// 		Key:    aws.String(key),
+	// 		Body:   f,
+	// 	})
+
+	return err
+}
+
+// serveFileFromS3 streams an object from S3 directly to the HTTP response.
+// It assumes access control has already been performed by the caller.
+func (s *Server) serveFileFromS3(c *gin.Context, key string) {
+	if s.s3Client == nil {
+		c.JSON(http.StatusInternalServerError, ErrorResponse{Error: "S3 storage not configured"})
+		return
+	}
+
+	input := &s3.GetObjectInput{
+		Bucket: aws.String(s.cfg.S3Bucket),
+		Key:    aws.String(key),
+	}
+	output, err := s.s3Client.GetObject(c.Request.Context(), input)
+	if err != nil {
+		golog.Errorf("s3 get object error: %v", err)
+		c.JSON(http.StatusNotFound, ErrorResponse{Error: "File not found"})
+		return
+	}
+	defer output.Body.Close()
+
+	// determine content type either from S3 metadata or fallback to octet-stream
+	contentType := "application/octet-stream"
+	if output.ContentType != nil {
+		contentType = *output.ContentType
+	}
+	c.Header("Content-Type", contentType)
+	// caching headers are handled by caller
+	if _, err := io.Copy(c.Writer, output.Body); err != nil {
+		golog.Errorf("error streaming s3 object: %v", err)
+	}
+}
+
 func writeFile(path, content string) error {
 	dir := filepath.Dir(path)
 	if err := os.MkdirAll(dir, 0755); err != nil {

go.mod

@@ -13,6 +13,29 @@ require (
 	modernc.org/sqlite v1.42.2
 )
 
+require (
+	github.com/aws/aws-sdk-go-v2 v1.41.3 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.32.11 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.11 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.5 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.19 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19 // indirect
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.96.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.0.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.12 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.41.8 // indirect
+	github.com/aws/smithy-go v1.24.2 // indirect
+)
+
 require (
 	cloud.google.com/go v0.116.0 // indirect
 	cloud.google.com/go/auth v0.14.0 // indirect