diff --git a/README.md b/README.md index 226e3c4..5013735 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,499 @@ -# shadcn/ui monorepo template +# BuildFlow πŸ”„ -This template is for creating a monorepo with shadcn/ui. +> A modern workflow automation platform with visual interface and event-driven execution engine. BuildFlow enables developers to build, deploy, and manage automated workflows connecting various services and APIs. -## Usage +## πŸ“‹ Table of Contents + +- [Overview](#overview) +- [Architecture](#architecture) +- [Tech Stack](#tech-stack) +- [Project Structure](#project-structure) +- [Database Schema](#database-schema) +- [Getting Started](#getting-started) +- [Development](#development) +- [Microservices](#microservices) +- [Key Features](#key-features) +- [Roadmap](#roadmap) + +## 🎯 Overview + +BuildFlow is a workflow automation platform inspired by N8N, designed to solve debugging and reliability issues in workflow execution. It provides a visual interface for creating workflows that connect triggers and actions (nodes) to automate business processes. + +### Core Concepts + +- **Workflows**: Visual representations of automated processes +- **Triggers**: Events that initiate workflow execution +- **Nodes**: Individual actions or operations within a workflow +- **Executions**: Tracked runs of workflows with detailed status and error handling +- **Credentials**: Secure OAuth and API key storage per integration + +## πŸ—οΈ Architecture + +BuildFlow follows a **microservices architecture** with an **event-driven execution model**: + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Web App │────▢│ HTTP Backend │────▢│ Database β”‚ +β”‚ (Next.js) β”‚ β”‚ (Express) β”‚ β”‚ (PostgreSQL)β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β–Ό + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ Hooks API β”‚ + β”‚ (Express) β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β–Ό + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ Processor │────▢│ Kafka β”‚ + β”‚ (KafkaJS) β”‚ β”‚ (Broker) β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β–Ό + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ Worker β”‚ + β”‚ (Consumer) β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +### Execution Flow + +1. **Trigger Event**: External event hits Hooks API (`/hooks/catch/:userId/:workflowId`) +2. **Transaction Outbox**: Workflow execution created atomically with outbox entry +3. **Event Publishing**: Processor reads outbox and publishes to Kafka +4. **Execution**: Worker consumes messages and executes workflow nodes +5. **Status Tracking**: Each node execution tracked with retry logic + +## πŸ› οΈ Tech Stack + +### Frontend +- **Framework**: Next.js 15.4.5 (App Router, Turbopack) +- **UI Library**: React 19.1.1 +- **State Management**: Redux Toolkit 2.11.2 with React Redux +- **Visual Editor**: React Flow (@xyflow/react) 12.9.3 +- **Authentication**: NextAuth.js 4.24.13 +- **Styling**: TailwindCSS 4.x with PostCSS +- **UI Components**: Radix UI, Lucide React icons +- **Validation**: Zod 3.25.76 +- **HTTP Client**: Axios 1.13.2 + +### Backend Services +- **HTTP Backend**: Express.js 5.1.0 (Port 3002) +- **Hooks Service**: Express.js 5.1.0 (Port 3002) +- **Processor**: KafkaJS 2.2.4 (Producer) +- **Worker**: KafkaJS 2.2.4 (Consumer) + +### Database & ORM +- **Database**: PostgreSQL +- **ORM**: Prisma Client +- **Migrations**: Prisma Migrate + +### Message Queue +- **Broker**: Apache Kafka +- **Client**: KafkaJS 2.2.4 +- **Topic**: `First-Client` + +### Development Tools +- **Monorepo**: Turborepo 2.5.5 +- **Package Manager**: pnpm 10.26.2 +- **TypeScript**: 5.7.3 / 5.9.2 +- **Linting**: ESLint 9.32.0 +- **Formatting**: Prettier 3.6.2 + +### Integrations +- **Google APIs**: googleapis 166.0.0 +- **OAuth**: google-auth-library 10.5.0 + +### Runtime Requirements +- **Node.js**: >=20 +- **Package Manager**: pnpm@10.26.2 + +## πŸ“ Project Structure + +``` +BuildFlow/ +β”œβ”€β”€ apps/ +β”‚ β”œβ”€β”€ web/ # Next.js frontend application +β”‚ β”‚ β”œβ”€β”€ app/ # App Router pages and routes +β”‚ β”‚ β”‚ β”œβ”€β”€ api/ # API routes (NextAuth, Google Sheets) +β”‚ β”‚ β”‚ β”œβ”€β”€ components/ # React components +β”‚ β”‚ β”‚ β”œβ”€β”€ hooks/ # Custom React hooks +β”‚ β”‚ β”‚ β”œβ”€β”€ workflow/ # Workflow editor page +β”‚ β”‚ β”‚ └── types/ # TypeScript types +β”‚ β”‚ └── store/ # Redux store configuration +β”‚ β”‚ +β”‚ β”œβ”€β”€ http-backend/ # Main Express API server +β”‚ β”‚ └── src/ +β”‚ β”‚ β”œβ”€β”€ routes/ # API route handlers +β”‚ β”‚ β”‚ β”œβ”€β”€ userRoutes/ # User management +β”‚ β”‚ β”‚ β”œβ”€β”€ nodes.routes.ts +β”‚ β”‚ β”‚ └── google_callback.ts +β”‚ β”‚ └── index.ts +β”‚ β”‚ +β”‚ β”œβ”€β”€ hooks/ # Webhook receiver service +β”‚ β”‚ └── src/index.ts # Receives external triggers +β”‚ β”‚ +β”‚ β”œβ”€β”€ processor/ # Kafka producer service +β”‚ β”‚ └── src/index.ts # Reads outbox, publishes to Kafka +β”‚ β”‚ +β”‚ └── worker/ # Kafka consumer service +β”‚ └── src/index.ts # Consumes messages, executes workflows +β”‚ +β”œβ”€β”€ packages/ +β”‚ β”œβ”€β”€ db/ # Prisma database package +β”‚ β”‚ └── prisma/ +β”‚ β”‚ └── schema.prisma # Database schema +β”‚ β”‚ +β”‚ β”œβ”€β”€ nodes/ # Node registry and executors +β”‚ β”‚ └── src/ +β”‚ β”‚ β”œβ”€β”€ google-sheets/ # Google Sheets integration +β”‚ β”‚ β”œβ”€β”€ common/ # Shared OAuth services +β”‚ β”‚ └── registry/ # Node registration system +β”‚ β”‚ +β”‚ β”œβ”€β”€ common/ # Shared utilities and schemas +β”‚ β”‚ └── src/index.ts # Zod schemas, constants +β”‚ β”‚ +β”‚ β”œβ”€β”€ ui/ # Shared UI component library +β”‚ β”‚ └── src/components/ # Reusable React components +β”‚ β”‚ +β”‚ β”œβ”€β”€ eslint-config/ # Shared ESLint configurations +β”‚ └── typescript-config/ # Shared TypeScript configurations +β”‚ +β”œβ”€β”€ turbo.json # Turborepo configuration +β”œβ”€β”€ pnpm-workspace.yaml # pnpm workspace configuration +└── package.json # Root package.json +``` + +## πŸ—„οΈ Database Schema + +### Core Models + +#### User Management +- **User**: User accounts with email/password authentication +- **Credential**: OAuth tokens and API keys per integration type + +#### Workflow System +- **Workflow**: Main workflow definition with status tracking +- **Trigger**: Workflow trigger configuration (one per workflow) +- **AvailableTrigger**: Registry of available trigger types +- **Node**: Individual action nodes in a workflow +- **AvailableNode**: Registry of available node types + +#### Execution Tracking +- **WorkflowExecution**: Tracks entire workflow runs +- **NodeExecution**: Tracks individual node executions with retry logic +- **WorkflowExecutionTable**: Transaction outbox pattern implementation + +### Workflow Status Enum +```typescript +enum WorkflowStatus { + Start // Initial state + Pending // Queued for execution + InProgress // Currently executing + ReConnecting // Retrying connection + Failed // Execution failed + Completed // Successfully completed +} +``` + +### Key Relationships +- User β†’ Workflows (1:N) +- Workflow β†’ Trigger (1:1) +- Workflow β†’ Nodes (1:N) +- WorkflowExecution β†’ NodeExecution (1:N) +- Node β†’ Credential (N:1, optional) + +## πŸš€ Getting Started + +### Prerequisites + +- **Node.js**: >=20 +- **pnpm**: 10.26.2 +- **PostgreSQL**: Running instance +- **Apache Kafka**: Running broker (default: localhost:9092) + +### Installation + +1. **Clone the repository** + ```bash + git clone + cd BuildFlow + ``` + +2. **Install dependencies** + ```bash + pnpm install + ``` + +3. **Set up environment variables** + + Create `.env` files in respective directories: + + **Root `.env`**: + ```env + DATABASE_URL="postgresql://user:password@localhost:5432/buildflow" + NEXTAUTH_SECRET="your-secret-key" + NEXTAUTH_URL="http://localhost:3000" + GOOGLE_CLIENT_ID="your-google-client-id" + GOOGLE_CLIENT_SECRET="your-google-client-secret" + ``` + + **Kafka Configuration** (for processor/worker): + ```env + KAFKA_BROKERS="localhost:9092" + KAFKA_CLIENT_ID="buildflow-client" + ``` + +4. **Set up the database** + ```bash + cd packages/db + pnpm prisma generate + pnpm prisma migrate dev + ``` + +5. **Start Kafka** (if not already running) + ```bash + # Using Docker + docker run -p 9092:9092 apache/kafka:latest + ``` + +### Running the Application + +#### Development Mode (All Services) ```bash -pnpm dlx shadcn@latest init +# Start all services in development mode +pnpm dev ``` -## Adding components +This will start: +- Web app: `http://localhost:3000` +- HTTP Backend: `http://localhost:3002` +- Hooks API: `http://localhost:3002` (same server) +- Processor: Background service +- Worker: Background service -To add components to your app, run the following command at the root of your `web` app: +#### Individual Services ```bash -pnpm dlx shadcn@latest add button -c apps/web +# Web application only +cd apps/web && pnpm dev + +# HTTP Backend only +cd apps/http-backend && pnpm dev + +# Processor only +cd apps/processor && pnpm dev + +# Worker only +cd apps/worker && pnpm dev + +# Hooks service only +cd apps/hooks && pnpm dev ``` -This will place the ui components in the `packages/ui/src/components` directory. +#### Build for Production + +```bash +# Build all packages +pnpm build + +# Build specific package +cd apps/web && pnpm build +``` + +## πŸ’» Development + +### Monorepo Management + +BuildFlow uses **Turborepo** for monorepo orchestration and **pnpm workspaces** for dependency management. + +#### Available Scripts -## Tailwind +- `pnpm dev` - Start all services in development mode +- `pnpm build` - Build all packages +- `pnpm lint` - Lint all packages +- `pnpm format` - Format code with Prettier -Your `tailwind.config.ts` and `globals.css` are already set up to use the components from the `ui` package. +#### Workspace Packages -## Using components +- `@repo/db` - Database client +- `@repo/nodes` - Node registry and executors +- `@repo/common` - Shared utilities +- `@workspace/ui` - UI component library +- `@workspace/eslint-config` - ESLint configurations +- `@workspace/typescript-config` - TypeScript configurations -To use the components in your app, import them from the `ui` package. +### Adding New Nodes -```tsx -import { Button } from "@workspace/ui/components/button" +1. Create node definition in `packages/nodes/src/[node-name]/` +2. Implement executor class extending base executor +3. Register node in `NodeRegistry.registerAll()` +4. Node will be automatically available in the UI + +Example: +```typescript +// packages/nodes/src/my-node/my-node.node.ts +export class MyNode { + static async register() { + await NodeRegistry.register({ + name: "My Node", + type: "my-node", + description: "Does something", + config: {}, + requireAuth: false, + }); + } +} ``` + +## πŸ”§ Microservices + +### HTTP Backend (`apps/http-backend`) +- **Port**: 3002 +- **Purpose**: Main API server for web app +- **Routes**: + - `/user/*` - User management, authentication + - `/node/*` - Node operations + - `/google/*` - Google OAuth callbacks +- **Features**: CORS enabled, cookie-based auth, Node registry initialization + +### Hooks Service (`apps/hooks`) +- **Port**: 3002 (shared with HTTP Backend) +- **Purpose**: Receives external webhook triggers +- **Endpoint**: `POST /hooks/catch/:userId/:workflowId` +- **Pattern**: Transaction Outbox - creates workflow execution atomically + +### Processor Service (`apps/processor`) +- **Purpose**: Kafka producer +- **Function**: Polls `WorkflowExecutionTable` (outbox), publishes to Kafka +- **Topic**: `First-Client` +- **Pattern**: Transaction Outbox Pattern implementation + +### Worker Service (`apps/worker`) +- **Purpose**: Kafka consumer and workflow executor +- **Function**: Consumes messages, executes workflow nodes sequentially +- **Group ID**: `test-group` +- **Features**: Manual offset commits, error handling + +## ✨ Key Features + +### 1. Visual Workflow Editor +- Drag-and-drop interface using React Flow +- Real-time workflow visualization +- Node configuration panels +- Trigger and action node types + +### 2. Node Registry System +- Dynamic node registration +- Pluggable architecture +- OAuth credential management per node +- Node metadata (name, type, description, auth requirements) + +### 3. Event-Driven Execution +- Transaction Outbox Pattern for reliable event publishing +- Kafka-based message queue +- Asynchronous workflow execution +- Guaranteed delivery semantics + +### 4. Execution Tracking +- Workflow-level execution tracking +- Node-level execution tracking +- Retry logic with configurable attempts +- Status monitoring (Start, Pending, InProgress, Failed, Completed) +- Input/output data capture +- Error logging and debugging + +### 5. Authentication & Security +- NextAuth.js integration +- OAuth 2.0 support (Google) +- Per-node credential storage +- Secure credential management + +### 6. Google Sheets Integration +- OAuth-based authentication +- Read/write operations +- Sheet and tab management +- Token refresh handling + +## πŸ—ΊοΈ Roadmap + +### Completed βœ… +- [x] Database schema design +- [x] Microservices architecture +- [x] Event-driven execution system +- [x] Transaction Outbox Pattern +- [x] Node registry system +- [x] Google Sheets integration +- [x] Visual workflow editor (basic) +- [x] User authentication + +### In Progress 🚧 +- [ ] Enhanced visual workflow editor +- [ ] Real-time execution dashboard +- [ ] Improved error handling and debugging + +### Planned πŸ“‹ +- [ ] Slack integration +- [ ] Webhook triggers +- [ ] Email integration +- [ ] Database integration nodes +- [ ] Conditional logic nodes +- [ ] Workflow templates +- [ ] Team collaboration features +- [ ] Workflow versioning +- [ ] Advanced retry strategies +- [ ] Webhook authentication +- [ ] API rate limiting +- [ ] Workflow scheduling (cron) +- [ ] Workflow testing/debugging tools + +## πŸ“ Technical Highlights + +### Transaction Outbox Pattern +Ensures reliable event publishing by storing workflow executions in a database table (`WorkflowExecutionTable`) before publishing to Kafka. The processor service polls this table and publishes events, ensuring no events are lost even if Kafka is temporarily unavailable. + +### Node Execution Flow +1. Workflow execution created with status `Start` +2. Nodes executed sequentially based on `position` field +3. Each node execution tracked with: + - Input/output data (JSON) + - Execution timestamps + - Retry count + - Error messages +4. Status updated through lifecycle: `Start` β†’ `Pending` β†’ `InProgress` β†’ `Completed`/`Failed` + +### OAuth Flow +1. User initiates OAuth flow from UI +2. Redirected to Google OAuth consent screen +3. Callback handled at `/google/callback` +4. Tokens stored in `Credential` table with encryption +5. Tokens refreshed automatically when expired + +### Workflow Status Lifecycle +``` +Start β†’ Pending β†’ InProgress β†’ Completed + ↓ + Failed β†’ ReConnecting β†’ InProgress +``` + +## 🀝 Contributing + +See [CONTRIBUTING.md](./CONTRIBUTING.md) for guidelines on: +- Branch naming conventions +- Pull request process +- Code review requirements +- Issue reporting + +## πŸ“„ License + +[Add your license here] + +## πŸ‘₯ Authors + +- [Your Name/Team] + +--- + +**Built with ❀️ using Next.js, Express, Kafka, and PostgreSQL** diff --git a/apps/http-backend/src/index.ts b/apps/http-backend/src/index.ts index f693f78..407fbd2 100644 --- a/apps/http-backend/src/index.ts +++ b/apps/http-backend/src/index.ts @@ -13,7 +13,7 @@ import { googleAuth } from "./routes/google_callback.js"; const app = express() -const allowedOrigins = ['http://localhost:3000' , 'http://localhost:3001' ]; +const allowedOrigins = ['http://localhost:3000' , 'http://localhost:300/' ]; app.use(cors({ origin: allowedOrigins, credentials: true, diff --git a/apps/http-backend/src/routes/google_callback.ts b/apps/http-backend/src/routes/google_callback.ts index c291914..f920cf2 100644 --- a/apps/http-backend/src/routes/google_callback.ts +++ b/apps/http-backend/src/routes/google_callback.ts @@ -21,7 +21,7 @@ googleAuth.get('/callback', async(req: Request, res: Response)=>{ ); try { - const { tokens } = await oauth2.getToken(code); + const { tokens } = await oauth2.getToken(code); // Save tokens to database if userId (state) is provided if (state && typeof state === 'string') { diff --git a/apps/http-backend/src/routes/nodes.routes.ts b/apps/http-backend/src/routes/nodes.routes.ts index 5288a0f..85c68ad 100644 --- a/apps/http-backend/src/routes/nodes.routes.ts +++ b/apps/http-backend/src/routes/nodes.routes.ts @@ -4,79 +4,100 @@ import { statusCodes } from "@repo/common/zod"; import { GoogleSheetsNodeExecutor } from "@repo/nodes"; export const sheetRouter: Router = Router(); -const sheetExecutor = new GoogleSheetsNodeExecutor() +const sheetExecutor = new GoogleSheetsNodeExecutor(); - -sheetRouter.get("/getDocuments/:cred", userMiddleware, async(req: AuthRequest, res: Response)=>{ - try{ - if (!req.user) - return res - .status(statusCodes.BAD_GATEWAY) - .json({ message: "User isnot logged in /not authorized" }); - const credId = req.params.cred - if(!credId){ - return res - .status(statusCodes.BAD_REQUEST) - .json({message: "credentials id not provided"}) - } - const userId = req.user.sub - console.log("userid from node route: ", userId) - if(!userId) return res - .status(statusCodes.NOT_FOUND) - .json({message: "User id not provided"}) - const sheets = await sheetExecutor.getSheets({userId: userId, credId: credId}) - if ((sheets as any)?.success === false) { - return res.status(statusCodes.NOT_FOUND).json({ - message: "files not found", - files: sheets - }); - } - return res.status(statusCodes.OK).json({ - message: "sheets are fetched successfully", - files: (sheets as any)?.data?.data?.files || [] - }) - }catch(e){ - console.log("Error Fetching the credentials ", e instanceof Error ? e.message : "Unkown reason"); +sheetRouter.get( + "/getDocuments/:cred", + userMiddleware, + async (req: AuthRequest, res: Response) => { + try { + if (!req.user) + return res + .status(statusCodes.BAD_GATEWAY) + .json({ message: "User isnot logged in /not authorized" }); + const credId = req.params.cred; + if (!credId) { + return res + .status(statusCodes.BAD_REQUEST) + .json({ message: "credentials id not provided" }); + } + const userId = req.user.sub; + console.log("userid from node route: ", userId); + if (!userId) + return res + .status(statusCodes.NOT_FOUND) + .json({ message: "User id not provided" }); + const sheets = await sheetExecutor.getSheets({ + userId: userId, + credId: credId, + }); + if ((sheets as any)?.success === false) { + return res.status(statusCodes.NOT_FOUND).json({ + message: "files not found", + files: sheets, + }); + } + return res.status(statusCodes.OK).json({ + message: "sheets are fetched successfully", + files: (sheets as any)?.data?.data?.files || [], + }); + } catch (e) { + console.log( + "Error Fetching the credentials ", + e instanceof Error ? e.message : "Unkown reason" + ); return res .status(statusCodes.INTERNAL_SERVER_ERROR) .json({ message: "Internal server from fetching the credentials" }); } -}) + } +); + +sheetRouter.get( + "/getSheets/:cred/:sheetId", + userMiddleware, + async (req: AuthRequest, res: Response) => { + try { + const userId = req.user?.sub; + if (!userId) + return res + .status(statusCodes.BAD_GATEWAY) + .json({ message: "User isnot logged in /not authorized" }); + const credId = req.params.cred; + const sheetId = req.params.sheetId; + if (!credId || !sheetId) { + return res + .status(statusCodes.BAD_REQUEST) + .json({ message: `credentials id not provided ` }); + } + if (!sheetExecutor) { + return res.status(statusCodes.FORBIDDEN).json({ + message: "sheet executor not configured well", + }); + } + const sheets = await sheetExecutor.getSheetTabs( + { userId: userId, credId: credId }, + sheetId + ); + + if ((sheets as any)?.success === false) + return res.status(statusCodes.NOT_FOUND).json({ + message: "files not found", + files: sheets, + }); -sheetRouter.get('/getSheets/:cred/:sheetId', userMiddleware, async(req: AuthRequest, res: Response)=>{ - try{ - const userId = req.user?.sub - if (!userId) - return res - .status(statusCodes.BAD_GATEWAY) - .json({ message: "User isnot logged in /not authorized" }); - const credId = req.params.cred - const sheetId = req.params.sheetId - if(!credId || !sheetId){ - return res - .status(statusCodes.BAD_REQUEST) - .json({message: `credentials id not provided `}) - } - if(!sheetExecutor){ - return res.status(statusCodes.FORBIDDEN).json({ - message: "sheet executor not configured well" - }) - } - const sheets = await sheetExecutor.getSheetTabs({userId: userId, credId: credId}, sheetId) - - if((sheets as any)?.success === false) return res.status(statusCodes.NOT_FOUND).json({ - message: "files not found", - files: sheets - }); - - return res.status(statusCodes.OK).json({ - message: "sheets are fetched successfully", - files: sheets - }) - }catch(e){ - console.log("Error Fetching the credentials ", e instanceof Error ? e.message : "Unkown reason"); + return res.status(statusCodes.OK).json({ + message: "sheets are fetched successfully", + files: sheets, + }); + } catch (e) { + console.log( + "Error Fetching the credentials ", + e instanceof Error ? e.message : "Unkown reason" + ); return res .status(statusCodes.INTERNAL_SERVER_ERROR) .json({ message: "Internal server from fetching the credentials" }); } -}) \ No newline at end of file + } +); diff --git a/apps/http-backend/src/routes/userRoutes/userMiddleware.ts b/apps/http-backend/src/routes/userRoutes/userMiddleware.ts index 6a1b84d..1df823c 100644 --- a/apps/http-backend/src/routes/userRoutes/userMiddleware.ts +++ b/apps/http-backend/src/routes/userRoutes/userMiddleware.ts @@ -21,7 +21,7 @@ export async function userMiddleware( next: NextFunction ) { try { - console.log(req.cookies); + // console.log(req.cookies); const payload = await getToken({ req, secret: SECRET }); if (!payload) { diff --git a/apps/http-backend/src/routes/userRoutes/userRoutes.ts b/apps/http-backend/src/routes/userRoutes/userRoutes.ts index 6cf40f5..d4e11a2 100644 --- a/apps/http-backend/src/routes/userRoutes/userRoutes.ts +++ b/apps/http-backend/src/routes/userRoutes/userRoutes.ts @@ -68,7 +68,8 @@ router.post("/createAvaliableNode", async (req: AuthRequest, res: Response) => { } }); -router.get("/getAvailableNodes", +router.get( + "/getAvailableNodes", userMiddleware, async (req: AuthRequest, res: Response) => { if (!req.user) { @@ -93,7 +94,8 @@ router.get("/getAvailableNodes", } ); -router.post("/createAvaliableTriggers", +router.post( + "/createTriggers", userMiddleware, async (req: AuthRequest, res: Response) => { try { @@ -127,7 +129,8 @@ router.post("/createAvaliableTriggers", } ); -router.get("/getAvailableTriggers", +router.get( + "/getAvailableTriggers", userMiddleware, async (req: AuthRequest, res: Response) => { try { @@ -155,44 +158,47 @@ router.get("/getAvailableTriggers", router.get('/getCredentials/:type', userMiddleware, - async (req: AuthRequest, res) =>{ - try{ - console.log("user from getcredentials: ",req.user) - if(!req.user){ - return res.status(statusCodes.BAD_REQUEST).json({ - message: "User is not Loggedin" - }) - } - const userId = req.user.sub; - const type = req.params.type - console.log(userId," -userid") - if(!type || !userId){ - return res.status(statusCodes.BAD_REQUEST).json({ - message: "Incorrect type Input", - }); - } - const executor = new GoogleSheetsNodeExecutor() - const response = await executor.getAllCredentials(userId,type) - // console.log( typeof(response)); - // console.log("response: ",response) - const authUrl = typeof response === 'string' ? response : null - // console.log(authUrl); - - const credentials = response instanceof Object ? response : null - // console.log(credentials) - if(authUrl){ - return res.status(statusCodes.OK).json({ - message: "Credentials not found create credentials using this auth url", + async (req: AuthRequest, res) => { + try { + console.log("user from getcredentials: ", req.user); + if (!req.user) { + return res.status(statusCodes.BAD_REQUEST).json({ + message: "User is not Loggedin", + }); + } + const userId = req.user.sub; + const type = req.params.type; + console.log(userId, " -userid"); + if (!type || !userId) { + return res.status(statusCodes.BAD_REQUEST).json({ + message: "Incorrect type Input", + }); + } + const executor = new GoogleSheetsNodeExecutor(); + const response = await executor.getAllCredentials(userId, type); + // console.log( typeof(response)); + // console.log("response: ",response) + const authUrl = typeof response === "string" ? response : null; + // console.log(authUrl); + + const credentials = response instanceof Object ? response : null; + // console.log(credentials) + if (authUrl) { + return res.status(statusCodes.OK).json({ + message: + "Credentials not found create credentials using this auth url", Data: authUrl, }); - } - else return res.status(statusCodes.OK).json({ + } else + return res.status(statusCodes.OK).json({ message: "Credentials Fetched succesfully", Data: credentials, }); - } - catch(e){ - console.log("Error Fetching the credentials ", e instanceof Error ? e.message : "Unkown reason"); + } catch (e) { + console.log( + "Error Fetching the credentials ", + e instanceof Error ? e.message : "Unkown reason" + ); return res .status(statusCodes.INTERNAL_SERVER_ERROR) .json({ message: "Internal server from fetching the credentials" }); @@ -200,9 +206,8 @@ router.get('/getCredentials/:type', } ); -// ----------------------------------- CREATE WORKFLOW --------------------------------- - -router.post("/create/workflow", +router.post( + "/create/workflow", userMiddleware, async (req: AuthRequest, res) => { try { @@ -252,9 +257,8 @@ router.post("/create/workflow", } ); -// ------------------------------------ FETCHING WORKFLOWS ----------------------------------- - -router.get("/workflows", +router.get( + "/workflows", userMiddleware, async (req: AuthRequest, res: Response) => { try { @@ -282,35 +286,8 @@ router.get("/workflows", } ); -router.get('/empty/workflow', userMiddleware, async(req:AuthRequest, res: Response)=>{ - try{ - if (!req.user) - return res - .status(statusCodes.UNAUTHORIZED) - .json({ message: "User is not logged in /not authorized" }); - const userId = req.user.id; - const workflow = await prismaClient.workflow.findFirst({ - where:{ - userId: userId, - isEmpty: true - }, - orderBy: { - createdAt: 'desc' - } - }) - return res - .status(statusCodes.OK) - .json({ message: "Workflow fetched succesful", Data: workflow }); - - }catch(e){ - console.log("The error is from getting wrkflows", e instanceof Error ? e.message : "UNKNOWN ERROR"); - - return res.status(statusCodes.INTERNAL_SERVER_ERROR).json({ - meesage: "Internal Server Error From getting workflows for the user", - }); - } -}) -router.get("/workflow/:workflowId", +router.get( + "/workflow/:workflowId", userMiddleware, async (req: AuthRequest, res: Response) => { try { diff --git a/apps/http-backend/tsconfig.tsbuildinfo b/apps/http-backend/tsconfig.tsbuildinfo index ed1b0ca..6d7fa0f 100644 --- a/apps/http-backend/tsconfig.tsbuildinfo +++ b/apps/http-backend/tsconfig.tsbuildinfo @@ -1 +1 @@ -{"root":["./src/index.ts","./src/routes/google_callback.ts","./src/routes/nodes.routes.ts","./src/routes/userroutes/usermiddleware.ts","./src/routes/userroutes/userroutes.ts"],"version":"5.7.3"} \ No newline at end of file +{"root":["./src/index.ts","./src/routes/google_callback.ts","./src/routes/nodes.routes.ts","./src/routes/userRoutes/userMiddleware.ts","./src/routes/userRoutes/userRoutes.ts"],"version":"5.7.3"} diff --git a/apps/web/app/components/nodes/CreateWorkFlow.tsx b/apps/web/app/components/nodes/CreateWorkFlow.tsx index 86b9b9a..1cceaab 100644 --- a/apps/web/app/components/nodes/CreateWorkFlow.tsx +++ b/apps/web/app/components/nodes/CreateWorkFlow.tsx @@ -1,7 +1,7 @@ "use client"; import { useEffect, useState } from "react"; import "@xyflow/react/dist/style.css"; -import { ReactFlow } from "@xyflow/react"; +import { Background, Controls, ReactFlow } from "@xyflow/react"; import PlaceholderNode from "./PlaceHolder"; import { TriggerNode } from "./TriggerNode"; @@ -15,7 +15,6 @@ import { createWorkflow, getEmptyWorkflow, getworkflowData } from "@/app/workflo import { useAppSelector } from '@/app/hooks/redux'; - interface NodeType { id: string; type: "placeholder" | "trigger" | "action"; @@ -39,8 +38,7 @@ export const CreateWorkFlow = () => { const [sidebarOpen, setSidebarOpen] = useState(false); const [actionSidebarOpen, setActionSidebarOpen] = useState(false); const [credType, setCredType] = useState(""); - const [nodeIDType, setNodeIDType] = useState('') - const [loadSheet, setLoadSheet] = useState(false) + const [loadSheet, setLoadSheet] = useState(false); const dispatch = useDispatch(); const userId = useAppSelector(s=>s.user.userId) const workflowId = useAppSelector(s=>s.workflow.workflow_id) @@ -119,35 +117,20 @@ export const CreateWorkFlow = () => { ]); }; - useEffect(()=>{ - async function getEmptyWorkflowID(){ - const workflow = await getEmptyWorkflow() - - if(workflow){ - const {id, isEmpty} = workflow - dispatch(workflowActions.setWorkflowId(id)) - dispatch(workflowActions.setWorkflowStatus(isEmpty)) - } - else{ - if (!userId) return - const newWorkflow = await createWorkflow(userId) - dispatch(workflowActions.setWorkflowId(newWorkflow.id)) - dispatch(workflowActions.setWorkflowStatus(newWorkflow.isEmpty)) - } - } - async function getWorkflowData(){ - if(!workflowId) return - const workflow = await getworkflowData(workflowId) - if(workflow.success){ - dispatch(workflowActions.setWorkflowStatus(false)) - dispatch(workflowActions.setWorkflowNodes(workflow.data.nodes)) - dispatch(workflowActions.setWorkflowTrigger(workflow.data.Trigger)) - // console.log(`workfklow from redux: ${workflow.data}`) + useEffect(() => { + async function getEmptyWorkflowID() { + const workflow = await getEmptyWorkflow(); + + if (workflow) { + const { id, isEmpty } = workflow; + dispatch(workflowActions.setWorkflowId(id)); + dispatch(workflowActions.setWorkflowStatus(isEmpty)); + } else { + // const newWorkflow } } - if(!workflowId) getEmptyWorkflowID() - getWorkflowData(); - },[dispatch, userId, workflowId]) + getEmptyWorkflowID(); + }, [dispatch]); const handleSelectAction = (action: { id: string; @@ -226,7 +209,7 @@ export const CreateWorkFlow = () => { nodeTypes={nodeTypes} nodes={nodes} edges={edges} - onNodeClick={async(event, node) => { + onNodeClick={async (event, node) => { if (node.type === "placeholder") { const hasTrigger = nodes.some((n) => n.type === "trigger"); if (hasTrigger) { @@ -235,22 +218,26 @@ export const CreateWorkFlow = () => { setSidebarOpen(true); } } - if(node.type === 'action' || node.type === 'trigger'){ - if(node.data.name === 'Google Sheet' ){ - console.log("sheet called") - console.log(node.id) + if (node.type === "action" || node.type === "trigger") { + if (node.data.name === "Google Sheet") { + console.log("sheet called"); + console.log(node.id); // setNodeId("550e8400-e29b-41d4-a716-446655440000") // getCredentials(node.data.type ? node.data.type : "") // setNodeId(node.id.split("trigger-")[1] || "") // if(cred) setLoadSheet(true) - setNodeIDType(node.id) - setCredType(node.data.type === "google_sheet" ? "google_oauth" : "") - setLoadSheet(!loadSheet) - console.log("hook called") + setCredType( + node.data.type === "google_sheet" ? "google_oauth" : "" + ); + setLoadSheet(!loadSheet); + console.log("hook called"); } - } + } }} - /> + > + + + { onSelectAction={handleSelectAction} /> - {loadSheet && - - } + {loadSheet && } ); }; diff --git a/apps/web/app/hooks/useCredential.ts b/apps/web/app/hooks/useCredential.ts index 565a24b..24d6b11 100644 --- a/apps/web/app/hooks/useCredential.ts +++ b/apps/web/app/hooks/useCredential.ts @@ -1,33 +1,34 @@ -"use client" -import { useEffect, useState } from "react" -import { getCredentials } from "../workflow/lib/config" +"use client"; +import { useEffect, useState } from "react"; +import { getCredentials } from "../workflow/lib/config"; -export const useCredentials = (type: string): any=>{ - const [cred, setCred] = useState() - const [authUrl, setAuthUrl] = useState() - useEffect(()=>{ - const fetchCred = async()=>{ - try{ - if(!type) return {} - const response = await getCredentials(type) - if(response){ - console.log(typeof(response)) - if(typeof(response) === 'string') setAuthUrl(response) - else setCred(response) - - // console.log(response[0].nodeId) - console.log(response) - return cred +export const useCredentials = (type: string): any => { + const [cred, setCred] = useState(); + const [authUrl, setAuthUrl] = useState(); + useEffect(() => { + const fetchCred = async () => { + try { + if (!type) return {}; + const response = await getCredentials(type); + if (response) { + console.log(typeof response); + if (typeof response === "string") setAuthUrl(response); + else setCred(response); - } - else return {} - } - catch(e){ - console.log(e instanceof Error ? e.message : "unknow error from useCredentials hook") - } - } + // console.log(response[0].nodeId) + console.log(response); + return cred; + } else return {}; + } catch (e) { + console.log( + e instanceof Error + ? e.message + : "unknow error from useCredentials hook" + ); + } + }; - fetchCred() - },[type]) - return {cred, authUrl} -} \ No newline at end of file + fetchCred(); + }, [type]); + return { cred, authUrl }; +}; diff --git a/apps/web/app/workflow/lib/config.ts b/apps/web/app/workflow/lib/config.ts index c877bce..c1d3645 100644 --- a/apps/web/app/workflow/lib/config.ts +++ b/apps/web/app/workflow/lib/config.ts @@ -1,6 +1,6 @@ import { BACKEND_URL } from "@repo/common/zod"; import { GoogleSheetsNodeExecutor } from "@repo/nodes"; -import axios from "axios" +import axios from "axios"; export const getAvailableTriggers = async () => { try { const response = await axios.get( @@ -11,7 +11,7 @@ export const getAvailableTriggers = async () => { ); console.log(response.data.Data); console.log(response.data); - + const Data = JSON.stringify(response.data.Data); console.log(Data); console.log("This is the name from the", response.data.Data); @@ -24,171 +24,20 @@ export const getAvailableTriggers = async () => { } }; -export const getCredentials = async(type: string)=>{ - try{ - +export const getCredentials = async (type: string) => { + try { const response = await axios.get( `${BACKEND_URL}/user/getCredentials/${type}`, { withCredentials: true, } ); - console.log("response from config: ",response); - + console.log("response from config: ", response); + const Data = JSON.stringify(response.data.Data); return response.data.Data; - } - catch(e){ + } catch (e) { console.error("Error fetching credentials:", e); throw e; } -} - -const date = new Date() - - -export const createWorkflow = async(userId: string)=>{ - try{ - const response = await axios.post(`${BACKEND_URL}/user/create/workflow`, - { - Name:`workflow-${date.getTime()}`, - UserId: userId, - Config: {} - },{ - headers: { - 'Content-Type': 'application/json' - }, - withCredentials: true - } - ) - - const workflow = response.data - console.log('new workflow: ',workflow) - return workflow.Data - }catch(e){ - console.error("Error in creating workflow:", e); - throw e; - } -} - -interface Triggercontext{ - // userId: formData.userId, - name: string, - workflowId: string, - node_Trigger: string, - config: { - credId: string, - operation: string, - spreadsheetId: string, - range: string, - sheetName: string, - }, - }; -export const createTrigger = async(context: Triggercontext)=>{ - try{ - console.log('Trigger context: ', context) - const response = await axios.post(`${BACKEND_URL}/user/create/trigger`, - { - Name: context.name, - AvailableTriggerID: context.node_Trigger, - Config: context.config, - WorkflowId: context.workflowId, - TriggerType:"" - }, - { - headers: {"Content-Type": "application/json"}, - withCredentials:true - }) - - const trigger = (response.data) - console.log('trigger created: ', trigger); - return { - success: true, - data: trigger - } - }catch(e){ - console.error("Error in creating Trigger:", e); - return { - success: false, - data: e instanceof Error ? e.message : `unknown error ${e}` - } - } -} - -interface Nodecontext{ - // userId: formData.userId, - name: string, - workflowId: string, - node_Trigger: string, - position? :number, - config: { - credId: string, - operation: string, - spreadsheetId: string, - range: string, - sheetName: string, - }, - }; -export const createNode = async(context: Nodecontext)=>{ - try{ - console.log('Node context: ', context) - - const response = await axios.post(`${BACKEND_URL}/user/create/node`,{ - - Name: context.name, - AvailableNodeId: context.node_Trigger, - WorkflowId: context.workflowId, - Config: context.config, - Position: context.position ? context.position : 0 - },{ - headers: {"Content-Type": "application/json"}, - withCredentials:true - }) - - const node = (response.data) - console.log('Node created: ', node); - return { - success: true, - data: node - } - - }catch(e){ - console.error("Error in creating Node:", e); - return {success: false, - data: e instanceof Error ? e.message : `unknown error ${e}` - }; - } -} - -export const getEmptyWorkflow = async()=>{ - try{ - const res = await axios.get(`${BACKEND_URL}/user/empty/workflow`,{ - withCredentials: true - }) - - const workflow = res.data.Data - return workflow - } - catch(e){ - console.error("Error in fetching workflows:", e); - throw e; - } -} - -export const getworkflowData = async(workflowId: string)=>{ - try{ - const res = await axios.get(`${BACKEND_URL}/user/workflow/${workflowId}`,{ - withCredentials: true - }) - console.log('workflow data includes trigger and nodes: ', res.data.Data ) - return { - success: true, - data: res.data.Data - } - }catch(e){ - console.error("Error in creating Node:", e); - return {success: false, - data: e instanceof Error ? e.message : `unknown error ${e}` - }; - } -} \ No newline at end of file +}; diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.ts index 099c219..c36ef78 100644 --- a/apps/worker/src/index.ts +++ b/apps/worker/src/index.ts @@ -21,6 +21,7 @@ async function main() { partition, offset: message.offset, value: message.value?.toString(), + topic }); await new Promise((r) => setTimeout(r, 5000));