Skip to content

[feat]: weaviate db configuration with population scripts #83

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
chandansgowda merged 3 commits into from
Jun 23, 2025
Merged

[feat]: weaviate db configuration with population scripts #83

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
60262c6
[refactor]: remove vectorizer dependency
smokeyScraper Jun 23, 2025
f10c790
[feat]: weaviate db models alignment to support user indexing and search
smokeyScraper Jun 23, 2025
6070611
[chore]: add weaviate db instance creation and population scripts
smokeyScraper Jun 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
103 changes: 78 additions & 25 deletions backend/app/model/weaviate/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,35 +1,88 @@
from pydantic import BaseModel, Field
from typing import List
from typing import List, Optional
from datetime import datetime


class WeaviateUserProfile(BaseModel):
class WeaviateRepository(BaseModel):
"""
Represents a vectorized user profile for semantic search in Weaviate.
Represents a single repostiory within WeaviateUserProfile.
Helps in structuring the repository-specific data that contributes to the user's overall profile
"""
supabase_user_id: str = Field(..., alias="supabaseUserId")
profile_summary: str = Field(..., alias="profileSummary")
primary_languages: List[str] = Field(..., alias="primaryLanguages")
expertise_areas: List[str] = Field(..., alias="expertiseAreas")
embedding: List[float] = Field(..., description="384-dimensional vector")
name: str = Field(..., description="The name of the repository.")
description: Optional[str] = Field(None, description="The repository's description.")
url: str = Field(..., description="The URL of the repository.")
languages: List[str] = Field(..., description="The languages used in the repository.")
stars: int = Field(0, description="The number of stars the repository has.")
forks: int = Field(0, description="The number of forks the repository has.")


class WeaviateCodeChunk(BaseModel):
class WeaviateUserProfile(BaseModel):
"""
Vectorized representation of code chunks stored in Weaviate.
Represents a user's profile data to be stored and indexed in Weaviate.
Enables semantic search capabilities to find users based on their profile data.
"""
supabase_chunk_id: str = Field(..., alias="supabaseChunkId")
code_content: str = Field(..., alias="codeContent")
language: str
function_names: List[str] = Field(..., alias="functionNames")
embedding: List[float] = Field(..., description="384-dimensional vector")
user_id: str = Field(..., description="The unique identifier for the user, linking back to the Supabase 'users' table.")
github_username: str = Field(..., description="The user's unique GitHub username.")
display_name: Optional[str] = Field(None, description="User's display name.")
bio: Optional[str] = Field(None, description="User's biography from their GitHub profile.")
location: Optional[str] = Field(None, description="User's location.")

repositories: List[WeaviateRepository] = Field(
default_factory=list, description="List of repositories the user's repositories.")

class WeaviateInteraction(BaseModel):
"""
Vectorized interaction representation stored in Weaviate.
"""
supabase_interaction_id: str = Field(..., alias="supabaseInteractionId")
conversation_summary: str = Field(..., alias="conversationSummary")
platform: str
topics: List[str]
embedding: List[float] = Field(..., description="384-dimensional vector")
languages: List[str] = Field(default_factory=list,
description="A unique, aggregated list of all programming languages from the user's repositories.")
topics: List[str] = Field(default_factory=list,
description="A unique, aggregated list of all topics from the user's repositories.")

followers_count: int = Field(0, description="Number of followers the user has on GitHub.")
following_count: int = Field(0, description="Number of other users this user is following on GitHub.")
total_stars_received: int = Field(
0, description="Total number of stars received across all of the user's owned repositories.")
total_forks: int = Field(0, description="Total number of times the user's repositories have been forked.")

profile_text_for_embedding: str = Field(
..., description="A synthesized text field combining bio, repository names, descriptions, languages, and topics for vectorization.")

last_updated: datetime = Field(default_factory=datetime.now,
description="The date and time the profile was last updated.")

class Config:
"""
Pydantic model configuration.
"""
orm_mode = True
schema_extra = {
"example": {
"user_id": "a1b2c3d4-e5f6-7890-1234-567890abcdef",
"github_username": "jane-dev",
"display_name": "Jane Developer",
"bio": "Creator of innovative open-source tools. Full-stack developer with a passion for Rust and WebAssembly.",
"location": "Berlin, Germany",
"repositories": [
{
"name": "rust-web-framework",
"description": "A high-performance web framework for Rust.",
"languages": ["Rust", "TOML"],
"topics": ["rust", "webdev", "performance", "framework"],
"stars": 2500,
"forks": 400
},
{
"name": "data-viz-lib",
"description": "A declarative data visualization library for JavaScript.",
"languages": ["JavaScript", "TypeScript"],
"topics": ["data-visualization", "d3", "charts"],
"stars": 1200,
"forks": 150
}
],
"languages": ["Rust", "JavaScript", "TypeScript", "TOML"],
"topics": ["rust", "webdev", "performance", "framework", "data-visualization", "d3", "charts"],
"followers_count": 1800,
"following_count": 250,
"total_stars_received": 3700,
"total_forks": 550,
"profile_text_for_embedding": "Jane Developer, Creator of innovative open-source tools. Full-stack developer with a passion for Rust and WebAssembly. Repositories: rust-web-framework, A high-performance web framework for Rust. data-viz-lib, A declarative data visualization library for JavaScript. Languages: Rust, JavaScript, TypeScript. Topics: rust, webdev, performance, data-visualization.",
"last_updated": "2025-06-23T12:21:00Z"
}
}
59 changes: 26 additions & 33 deletions backend/app/scripts/weaviate/create_schemas.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,48 +1,41 @@
from app.db.weaviate.weaviate_client import get_client
import weaviate.classes.config as wc

def create_schema(client, name, properties):
client.collections.create(
name=name,
properties=properties,
vectorizer_config=wc.Configure.Vectorizer.text2vec_cohere(),
generative_config=wc.Configure.Generative.openai()
)
print(f"Created: {name}")
def create_user_profile_schema(client):
properties = [
wc.Property(name="supabaseUserId", data_type=wc.DataType.TEXT),
wc.Property(name="profileSummary", data_type=wc.DataType.TEXT),
wc.Property(name="primaryLanguages", data_type=wc.DataType.TEXT_ARRAY),
wc.Property(name="expertiseAreas", data_type=wc.DataType.TEXT_ARRAY),
]
create_schema(client, "weaviate_user_profile", properties)


def create_code_chunk_schema(client):
properties = [
wc.Property(name="supabaseChunkId", data_type=wc.DataType.TEXT),
wc.Property(name="codeContent", data_type=wc.DataType.TEXT),
wc.Property(name="language", data_type=wc.DataType.TEXT),
wc.Property(name="functionNames", data_type=wc.DataType.TEXT_ARRAY),
]
create_schema(client, "weaviate_code_chunk", properties)

def create_interaction_schema(client):
def create_user_profile_schema(client):
"""
Create schema for WeaviateUserProfile model.
Main vectorization will be on profile_text_for_embedding field.
"""
properties = [
wc.Property(name="supabaseInteractionId", data_type=wc.DataType.TEXT),
wc.Property(name="conversationSummary", data_type=wc.DataType.TEXT),
wc.Property(name="platform", data_type=wc.DataType.TEXT),
wc.Property(name="user_id", data_type=wc.DataType.TEXT),
wc.Property(name="github_username", data_type=wc.DataType.TEXT),
wc.Property(name="display_name", data_type=wc.DataType.TEXT),
wc.Property(name="bio", data_type=wc.DataType.TEXT),
wc.Property(name="location", data_type=wc.DataType.TEXT),
wc.Property(name="repositories", data_type=wc.DataType.TEXT), # JSON string
wc.Property(name="languages", data_type=wc.DataType.TEXT_ARRAY),
wc.Property(name="topics", data_type=wc.DataType.TEXT_ARRAY),
wc.Property(name="followers_count", data_type=wc.DataType.INT),
wc.Property(name="following_count", data_type=wc.DataType.INT),
wc.Property(name="total_stars_received", data_type=wc.DataType.INT),
wc.Property(name="total_forks", data_type=wc.DataType.INT),
wc.Property(name="profile_text_for_embedding", data_type=wc.DataType.TEXT),
wc.Property(name="last_updated", data_type=wc.DataType.DATE),
]
create_schema(client, "weaviate_interaction", properties)
create_schema(client, "weaviate_user_profile", properties)

def create_all_schemas():
"""
Create only the user profile schema as per the updated model structure.
"""
client = get_client()
existing_collections = client.collections.list_all()
if "weaviate_code_chunk" not in existing_collections:
create_code_chunk_schema(client)
if "weaviate_interaction" not in existing_collections:
create_interaction_schema(client)
if "weaviate_user_profile" not in existing_collections:
create_user_profile_schema(client)
print("✅ All schemas ensured.")
create_user_profile_schema(client)
client.close()
print("✅ User profile schema created successfully.")
Loading