Skip to content

Commit d8dd9bd

Browse files
rishhbhrishhbh
authored
Merge pull request #4 from qafariamirhossein/feat/streaming-ai-responses
feat: add streaming AI responses
2 parents ccf6844 + c70bec2 commit d8dd9bd

7 files changed

Lines changed: 397 additions & 61 deletions

File tree

‎client/src/lib/streamApi.ts‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
type StreamOptions = {
2+
onDelta: (delta: string) => void;
3+
};
4+
5+
const API_BASE_URL = `${import.meta.env.VITE_API_URL}/api`;
6+
7+
const readSSEStream = async (
8+
response: Response,
9+
onDelta: (delta: string) => void
10+
) => {
11+
if (!response.ok || !response.body) {
12+
let message = "Failed to stream response";
13+
14+
try {
15+
const data = await response.json();
16+
message = data.message || message;
17+
} catch {
18+
// ignore JSON parse errors
19+
}
20+
21+
throw new Error(message);
22+
}
23+
24+
const reader = response.body.getReader();
25+
const decoder = new TextDecoder();
26+
27+
let buffer = "";
28+
let finalText = "";
29+
30+
while (true) {
31+
const { value, done } = await reader.read();
32+
33+
if (done) break;
34+
35+
buffer += decoder.decode(value, { stream: true });
36+
37+
const events = buffer.split("\n\n");
38+
buffer = events.pop() || "";
39+
40+
for (const eventBlock of events) {
41+
const eventLine = eventBlock
42+
.split("\n")
43+
.find((line) => line.startsWith("event:"));
44+
45+
const dataLine = eventBlock
46+
.split("\n")
47+
.find((line) => line.startsWith("data:"));
48+
49+
if (!eventLine || !dataLine) continue;
50+
51+
const event = eventLine.replace("event:", "").trim();
52+
const payload = JSON.parse(dataLine.replace("data:", "").trim());
53+
54+
if (event === "chunk") {
55+
finalText += payload.delta || "";
56+
onDelta(payload.delta || "");
57+
}
58+
59+
if (event === "error") {
60+
throw new Error(payload.message || "Streaming failed");
61+
}
62+
}
63+
}
64+
65+
return finalText;
66+
};
67+
68+
export const postJsonStream = async (
69+
endpoint: string,
70+
body: Record<string, unknown>,
71+
options: StreamOptions
72+
) => {
73+
const response = await fetch(`${API_BASE_URL}${endpoint}`, {
74+
method: "POST",
75+
credentials: "include",
76+
headers: {
77+
"Content-Type": "application/json",
78+
},
79+
body: JSON.stringify({
80+
...body,
81+
stream: true,
82+
}),
83+
});
84+
85+
return readSSEStream(response, options.onDelta);
86+
};
87+
88+
export const postFormStream = async (
89+
endpoint: string,
90+
formData: FormData,
91+
options: StreamOptions
92+
) => {
93+
formData.append("stream", "true");
94+
95+
const response = await fetch(`${API_BASE_URL}${endpoint}`, {
96+
method: "POST",
97+
credentials: "include",
98+
body: formData,
99+
});
100+
101+
return readSSEStream(response, options.onDelta);
102+
};

‎client/src/pages/DocSummarizer.tsx‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import { Loader2, FileText } from 'lucide-react';
1212
import jsPDF from 'jspdf';
1313
import { marked } from 'marked';
1414
import { Download } from 'lucide-react';
15+
import { postFormStream } from "../lib/streamApi";
1516

1617
const DocSummarizer: React.FC = () => {
1718
const [file, setFile] = useState<File | null>(null);
@@ -26,24 +27,23 @@ const DocSummarizer: React.FC = () => {
2627
}
2728

2829
setIsLoading(true);
29-
setSummary(null);
30-
try {
31-
const formData = new FormData();
32-
formData.append('document', file);
33-
formData.append('client', client);
30+
setSummary("");
3431

35-
const res = await api.post('/scrape/doc', formData, {
36-
headers: {
37-
'Content-Type': 'multipart/form-data'
38-
}
39-
});
40-
const summaryContent = res.data.output || res.data.summary || res.data;
41-
setSummary(typeof summaryContent === 'string' ? summaryContent : JSON.stringify(summaryContent, null, 2));
42-
} catch (error: any /* eslint-disable-line @typescript-eslint/no-explicit-any */) {
43-
toast.error(error.response?.data?.message || "Failed to generate summary");
44-
} finally {
45-
setIsLoading(false);
46-
}
32+
try {
33+
const formData = new FormData();
34+
formData.append("document", file);
35+
formData.append("client", client);
36+
37+
await postFormStream("/scrape/doc", formData, {
38+
onDelta: (delta) => {
39+
setSummary((prev) => `${prev || ""}${delta}`);
40+
},
41+
});
42+
} catch (error: any) {
43+
toast.error(error.message || "Failed to generate summary");
44+
} finally {
45+
setIsLoading(false);
46+
}
4747
};
4848

4949
const downloadSummary = async (summary: string) => {

‎client/src/pages/UrlSummarizer.tsx‎

Lines changed: 19 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import { Loader2, Globe } from 'lucide-react';
1616
import jsPDF from 'jspdf';
1717
import { marked } from 'marked';
1818
import { Download } from 'lucide-react';
19+
import { postJsonStream } from "../lib/streamApi";
1920

2021
const urlSchema = z.object({
2122
url: z.string().url({ message: "Please enter a valid URL" }),
@@ -37,30 +38,26 @@ const UrlSummarizer: React.FC = () => {
3738

3839
const onSubmit = async (data: UrlFormValues) => {
3940
setIsLoading(true);
40-
setSummary(null);
41-
try {
42-
const res = await api.post('/scrape/web', {
43-
url: data.url,
44-
client: data.client
45-
});
41+
setSummary("");
4642

47-
const summaryContent = res.data.output || res.data.summary || res.data.content || res.data;
48-
setSummary(typeof summaryContent === 'string' ? summaryContent : JSON.stringify(summaryContent, null, 2));
49-
} catch (error: any /* eslint-disable-line @typescript-eslint/no-explicit-any */) {
50-
if (error.response?.data?.errors) {
51-
const backendErrors = error.response.data.errors;
52-
Object.keys(backendErrors).forEach((key) => {
53-
setError(key as any /* eslint-disable-line @typescript-eslint/no-explicit-any */, {
54-
type: "server",
55-
message: backendErrors[key][0],
56-
});
57-
});
58-
} else {
59-
toast.error(error.response?.data?.message || "Failed to generate summary");
60-
}
61-
} finally {
62-
setIsLoading(false);
43+
try {
44+
await postJsonStream(
45+
"/scrape/web",
46+
{
47+
url: data.url,
48+
client: data.client,
49+
},
50+
{
51+
onDelta: (delta) => {
52+
setSummary((prev) => `${prev || ""}${delta}`);
53+
},
6354
}
55+
);
56+
} catch (error: any) {
57+
toast.error(error.message || "Failed to generate summary");
58+
} finally {
59+
setIsLoading(false);
60+
}
6461
};
6562

6663
const downloadSummary = async (summary: string) => {

‎server/config/llm/streamingClients.js‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
import systemPrompt from "../systemPrompt.js";
2+
import sarvamSystemPrompt from "../sarvamSystemPrompt.js";
3+
4+
import { GoogleGenAI } from "@google/genai";
5+
import Cerebras from "@cerebras/cerebras_cloud_sdk";
6+
import { SarvamAIClient } from "sarvamai";
7+
8+
import sarvamClient from "./sarvamClient.js";
9+
10+
const geminiAI = new GoogleGenAI({
11+
apiKey: process.env.GEMINI_API_KEY,
12+
});
13+
14+
const cerebras = new Cerebras({
15+
apiKey: process.env.CEREBRAS_API_KEY,
16+
});
17+
18+
export async function* streamGemini(prompt) {
19+
const responseStream = await geminiAI.models.generateContentStream({
20+
model: "gemini-2.5-flash",
21+
contents: prompt,
22+
config: {
23+
systemInstruction: systemPrompt,
24+
temperature: 0.7,
25+
},
26+
});
27+
28+
for await (const chunk of responseStream) {
29+
const text = chunk.text || "";
30+
if (text) yield text;
31+
}
32+
}
33+
34+
export async function* streamCerebras(prompt) {
35+
const stream = await cerebras.chat.completions.create({
36+
model: "gpt-oss-120b",
37+
stream: true,
38+
messages: [
39+
{ role: "system", content: systemPrompt },
40+
{ role: "user", content: prompt },
41+
],
42+
});
43+
44+
for await (const chunk of stream) {
45+
const text = chunk.choices?.[0]?.delta?.content || "";
46+
if (text) yield text;
47+
}
48+
}
49+
50+
export async function* streamGemma(prompt) {
51+
const response = await fetch(`${process.env.OLLAMA_BASE_URL}/api/chat`, {
52+
method: "POST",
53+
headers: {
54+
"Content-Type": "application/json",
55+
},
56+
body: JSON.stringify({
57+
model: process.env.OLLAMA_MODEL,
58+
messages: [
59+
{ role: "system", content: systemPrompt },
60+
{ role: "user", content: prompt },
61+
],
62+
stream: true,
63+
}),
64+
});
65+
66+
if (!response.ok || !response.body) {
67+
throw new Error("Failed to stream response from Ollama");
68+
}
69+
70+
const reader = response.body.getReader();
71+
const decoder = new TextDecoder();
72+
let buffer = "";
73+
74+
while (true) {
75+
const { value, done } = await reader.read();
76+
77+
if (done) break;
78+
79+
buffer += decoder.decode(value, { stream: true });
80+
81+
const lines = buffer.split("\n");
82+
buffer = lines.pop() || "";
83+
84+
for (const line of lines) {
85+
if (!line.trim()) continue;
86+
87+
const json = JSON.parse(line);
88+
const text = json.message?.content || "";
89+
90+
if (text) yield text;
91+
if (json.done) return;
92+
}
93+
}
94+
}
95+
96+
export async function* streamSarvam(prompt) {
97+
const client = new SarvamAIClient({
98+
apiSubscriptionKey: process.env.SARVAM_API_KEY,
99+
});
100+
101+
try {
102+
const stream = await client.chat.completions({
103+
model: "sarvam-30b",
104+
stream: true,
105+
messages: [
106+
{ role: "system", content: sarvamSystemPrompt },
107+
{
108+
role: "user",
109+
content: `${prompt}\n\n(Reply in Hinglish, same energy as always)`,
110+
},
111+
],
112+
temperature: 0.5,
113+
top_p: 1,
114+
max_tokens: 2000,
115+
reasoning_effort: null,
116+
});
117+
118+
for await (const chunk of stream) {
119+
const text = chunk.choices?.[0]?.delta?.content || "";
120+
if (text) yield text;
121+
}
122+
} catch {
123+
// Safe fallback if the installed Sarvam JS SDK version does not support streaming.
124+
const fullResponse = await sarvamClient(prompt);
125+
yield fullResponse;
126+
}
127+
}
128+
129+
export const streamingModels = {
130+
gemini: streamGemini,
131+
cerebras: streamCerebras,
132+
gemma: streamGemma,
133+
sarvam: streamSarvam,
134+
};

0 commit comments

Comments
 (0)