-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathlogger.py
More file actions
429 lines (344 loc) · 13.9 KB
/
Copy pathlogger.py
File metadata and controls
429 lines (344 loc) · 13.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
"""
FUSE - (python dev tool) - Core Logger
logger.py - Logger Infrastructure with Event Loop Handling
"""
import asyncio
import json
import logging
from collections import deque
from dataclasses import dataclass, asdict
from datetime import datetime
from enum import Enum
from pathlib import Path
from typing import List, Dict, Optional, AsyncGenerator, Any
from logging.handlers import RotatingFileHandler
import structlog
import colorama
from colorama import Fore, Style
# Initialize colorama for cross-platform colored output
colorama.init()
class LogLevel(Enum):
"""Log levels with color coding"""
DEBUG = ("DEBUG", Fore.CYAN)
INFO = ("INFO", Fore.GREEN)
WARNING = ("WARNING", Fore.YELLOW)
ERROR = ("ERROR", Fore.RED)
CRITICAL = ("CRITICAL", Fore.MAGENTA)
@dataclass
class LogEntry:
"""Log entry data structure"""
timestamp: datetime
level: str
message: str
source: str
project_id: str
process_id: Optional[str] = None
metadata: Optional[Dict[str, Any]] = None
exception: Optional[str] = None
def to_dict(self) -> Dict[str, Any]:
"""Convert to dictionary for JSON serialization"""
return {
**asdict(self),
"timestamp": self.timestamp.isoformat(),
}
def to_json(self) -> str:
"""Convert to JSON string"""
return json.dumps(self.to_dict())
class ColoredFormatter(logging.Formatter):
"""Custom colored formatter for console output"""
COLORS = {
"DEBUG": Fore.CYAN,
"INFO": Fore.GREEN,
"WARNING": Fore.YELLOW,
"ERROR": Fore.RED,
"CRITICAL": Fore.MAGENTA,
}
def format(self, record):
# Get color for log level
level_color = self.COLORS.get(record.levelname, Fore.WHITE)
# Format timestamp
timestamp = datetime.fromtimestamp(record.created).strftime("%H:%M:%S")
# Format the message
formatted = (
f"{Fore.BLUE}[{timestamp}]{Style.RESET_ALL} "
f"{level_color}[{record.levelname:8}]{Style.RESET_ALL} "
f"{Fore.WHITE}[{record.name}]{Style.RESET_ALL} "
f"{record.getMessage()}"
)
# Add exception info if present
if record.exc_info:
formatted += f"\n{self.formatException(record.exc_info)}"
return formatted
class LogBuffer:
"""Thread-safe circular buffer for log entries"""
def __init__(self, max_size: int = 10000):
self.buffer = deque(maxlen=max_size)
self.lock = asyncio.Lock()
self.subscribers = set()
async def add_entry(self, entry: LogEntry):
"""Add log entry to buffer"""
async with self.lock:
self.buffer.append(entry)
# Notify subscribers
await self._notify_subscribers(entry)
def add_entry_sync(self, entry: LogEntry):
"""Add log entry to buffer synchronously (for when no event loop is running)"""
self.buffer.append(entry)
# Note: Can't notify subscribers in sync mode
async def get_recent(self, count: int = 100) -> List[LogEntry]:
"""Get recent log entries"""
async with self.lock:
return list(self.buffer)[-count:]
async def get_filtered(
self,
level: Optional[str] = None,
source: Optional[str] = None,
since: Optional[datetime] = None,
count: int = 100,
) -> List[LogEntry]:
"""Get filtered log entries"""
async with self.lock:
entries = list(self.buffer)
# Apply filters
if level:
entries = [e for e in entries if e.level == level]
if source:
entries = [e for e in entries if e.source == source]
if since:
entries = [e for e in entries if e.timestamp >= since]
return entries[-count:]
def subscribe(self, callback):
"""Subscribe to new log entries"""
self.subscribers.add(callback)
def unsubscribe(self, callback):
"""Unsubscribe from log entries"""
self.subscribers.discard(callback)
async def _notify_subscribers(self, entry: LogEntry):
"""Notify all subscribers of new entry"""
for callback in self.subscribers:
try:
if asyncio.iscoroutinefunction(callback):
await callback(entry)
else:
callback(entry)
except Exception as e:
# Log error but don't propagate to avoid breaking the logger
print(f"Error in log subscriber: {e}")
class LogManager:
"""Centralized log management system"""
def __init__(self, project_id: str, log_level: str = "INFO"):
self.project_id = project_id
self.log_level = getattr(logging, log_level.upper())
# Create logs directory
self.log_dir = Path(f".fuse/logs")
self.log_dir.mkdir(parents=True, exist_ok=True)
# Log files
self.log_file = self.log_dir / "app.log"
self.error_log_file = self.log_dir / "error.log"
# Initialize buffer
self.buffer = LogBuffer()
# Setup standard logging first
self._setup_standard_logging()
# Setup structured logging
self._setup_structured_logging()
# Logger instances
self.loggers: Dict[str, logging.Logger] = {}
self.main_logger = self.get_logger("main")
def _setup_structured_logging(self):
"""Setup structured logging with structlog"""
structlog.configure(
processors=[
structlog.stdlib.filter_by_level,
structlog.stdlib.add_logger_name,
structlog.stdlib.add_log_level,
structlog.stdlib.PositionalArgumentsFormatter(),
structlog.processors.TimeStamper(fmt="ISO"),
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
structlog.processors.UnicodeDecoder(),
self._structlog_processor,
structlog.processors.JSONRenderer(),
],
context_class=dict,
logger_factory=structlog.stdlib.LoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=True,
)
def _setup_standard_logging(self):
"""Setup standard Python logging"""
# Create formatters
file_formatter = logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s"
)
console_formatter = ColoredFormatter()
# Setup file handler with rotation
file_handler = RotatingFileHandler(
self.log_file,
maxBytes=10 * 1024 * 1024, # 10MB
backupCount=5,
)
file_handler.setFormatter(file_formatter)
file_handler.setLevel(self.log_level)
# Setup error file handler
error_handler = RotatingFileHandler(
self.error_log_file,
maxBytes=10 * 1024 * 1024, # 10MB
backupCount=5,
)
error_handler.setFormatter(file_formatter)
error_handler.setLevel(logging.ERROR)
# Setup console handler
console_handler = logging.StreamHandler()
console_handler.setFormatter(console_formatter)
console_handler.setLevel(self.log_level)
# Configure root logger
root_logger = logging.getLogger()
root_logger.setLevel(self.log_level)
root_logger.addHandler(file_handler)
root_logger.addHandler(error_handler)
root_logger.addHandler(console_handler)
def _structlog_processor(self, logger, method_name, event_dict):
"""Custom structlog processor to add entries to buffer"""
# Create log entry
entry = LogEntry(
timestamp=datetime.now(),
level=event_dict.get("level", "INFO").upper(),
message=event_dict.get("event", ""),
source=event_dict.get("logger", "unknown"),
project_id=self.project_id,
process_id=event_dict.get("process_id"),
metadata=event_dict.get("metadata", {}),
exception=event_dict.get("exception"),
)
# Try to add to buffer asynchronously if there's a running event loop
try:
# Check if there's a running event loop
loop = asyncio.get_running_loop()
# If we get here, there's a running loop, so we can create a task
asyncio.create_task(self.buffer.add_entry(entry))
except RuntimeError:
# No running event loop, add synchronously
self.buffer.add_entry_sync(entry)
return event_dict
def get_logger(self, name: str) -> logging.Logger:
"""Get or create logger for given name"""
if name not in self.loggers:
logger = logging.getLogger(f"fuse.{self.project_id}.{name}")
self.loggers[name] = logger
return self.loggers[name]
def get_structured_logger(self, name: str):
"""Get structured logger"""
return structlog.get_logger(f"fuse.{self.project_id}.{name}")
async def get_logs(
self,
level: Optional[str] = None,
source: Optional[str] = None,
since: Optional[datetime] = None,
count: int = 100,
) -> List[LogEntry]:
"""Get recent logs with filtering"""
return await self.buffer.get_filtered(level, source, since, count)
async def stream_logs(self) -> AsyncGenerator[LogEntry, None]:
"""Stream logs in real-time"""
queue = asyncio.Queue()
async def subscriber(entry: LogEntry):
await queue.put(entry)
self.buffer.subscribe(subscriber)
try:
while True:
entry = await queue.get()
yield entry
finally:
self.buffer.unsubscribe(subscriber)
async def clear_logs(self):
"""Clear log buffer"""
async with self.buffer.lock:
self.buffer.buffer.clear()
def log_process_start(self, process_id: str, command: str):
"""Log process start"""
self.main_logger.info(
f"Process started: {command}",
extra={"process_id": process_id, "command": command},
)
def log_process_stop(self, process_id: str, exit_code: int):
"""Log process stop"""
level = logging.INFO if exit_code == 0 else logging.ERROR
self.main_logger.log(
level,
f"Process stopped with exit code: {exit_code}",
extra={"process_id": process_id, "exit_code": exit_code},
)
def log_file_change(self, file_path: str, change_type: str):
"""Log file change"""
self.main_logger.info(
f"File {change_type}: {file_path}",
extra={"file_path": file_path, "change_type": change_type},
)
def log_deployment_start(self, platform: str, deployment_id: str):
"""Log deployment start"""
self.main_logger.info(
f"Deployment started to {platform}",
extra={"platform": platform, "deployment_id": deployment_id},
)
def log_deployment_complete(self, platform: str, deployment_id: str, success: bool):
"""Log deployment completion"""
level = logging.INFO if success else logging.ERROR
status = "successful" if success else "failed"
self.main_logger.log(
level,
f"Deployment {status} to {platform}",
extra={
"platform": platform,
"deployment_id": deployment_id,
"success": success,
},
)
def log_error(self, error: Exception, context: Optional[Dict[str, Any]] = None):
"""Log error with context"""
self.main_logger.error(
f"Error: {str(error)}", exc_info=True, extra={"context": context or {}}
)
def close(self):
"""Close all handlers and cleanup"""
for handler in logging.getLogger().handlers[:]:
handler.close()
logging.getLogger().removeHandler(handler)
# Global log manager instance
_log_managers: Dict[str, LogManager] = {}
def get_log_manager(project_id: str, log_level: str = "INFO") -> LogManager:
"""Get or create log manager for project"""
if project_id not in _log_managers:
_log_managers[project_id] = LogManager(project_id, log_level)
return _log_managers[project_id]
def setup_logging(project_id: str, log_level: str = "INFO") -> LogManager:
"""Setup logging for project"""
return get_log_manager(project_id, log_level)
# Convenience functions
def get_logger(project_id: str, name: str) -> logging.Logger:
"""Get logger for project and name"""
return get_log_manager(project_id).get_logger(name)
def get_structured_logger(project_id: str, name: str):
"""Get structured logger for project and name"""
return get_log_manager(project_id).get_structured_logger(name)
async def get_project_logs(project_id: str, **kwargs) -> List[LogEntry]:
"""Get logs for project"""
return await get_log_manager(project_id).get_logs(**kwargs)
async def stream_project_logs(project_id: str) -> AsyncGenerator[LogEntry, None]:
"""Stream logs for project"""
async for entry in get_log_manager(project_id).stream_logs():
yield entry
class DevToolLogger:
"""Simplified logger interface for the dev tool"""
def __init__(self, project_id: str = "default", name: str = "main"):
# Used standard library logger instead of structlog for simplicity
self.logger = get_log_manager(project_id).get_logger(name)
def debug(self, message: str, **kwargs):
self.logger.debug(message, extra=kwargs)
def info(self, message: str, **kwargs):
self.logger.info(message, extra=kwargs)
def warning(self, message: str, **kwargs):
self.logger.warning(message, extra=kwargs)
def error(self, message: str, **kwargs):
self.logger.error(message, extra=kwargs)
def critical(self, message: str, **kwargs):
self.logger.critical(message, extra=kwargs)