Ronydahdal/simurgh

infra/docker-compose.yml

@@ -79,42 +79,42 @@ services: # this is where services are defined. A service is a docker container,
       retries: 5
   redis:
     image: redis:7.0.9
-  # backend: # this is the backend service, which is the main service of the application
-  #   depends_on: # In defining depends_on, we define what we want to wait for before starting the service.
-  #     postgres:
-  #       condition: service_healthy # there are multiple conditions, but service_healthy is the last one to be checked, so it is the most reliable.
-  #   ports:
-  #     - 3000:3000
-  #   environment:
-  #     - DATABASE_URL=postgres://postgres:postgrespw@postgres:5432/postgres
-  #     - LOGGING_URL=http://loki:3100/loki/api/v1/push
-  #     - PROMETHEUS_URL=http://prometheus:9090
-  #     - REDIS_URL=redis://redis:6379
-  #     - STATSD_URL=statsdexporter:9125
-  #     - PORT=3000
-  #     - SAML_CERT_PATH=/certs/cert.pem
-  #     - SAML_KEY_PATH=/certs/key.pem
-  #     - IDP_URL=http://host.docker.internal:8080/idp.xml #for prod: https://idp.elon.edu/idp/shibboleth
-  #     - SP_URL=http://localhost:3000
-  #     - WEB_URL=http://localhost:3001
-  #   volumes:
-  #     - ./saml:/certs
-  #     - ./saml/cert.pem:/certs/cert.pem
-  #     - ./saml/key.pem:/certs/key.pem
-  #   build: # In build, we define where to build the provided docker file. This is in contrast to just supplying the image which is queried from a registry. This service is local and thus not avaiable in a registry, so we have to build it locally.
-  #     context: ../backend/ # Context describes where to build the dockerfile from. If not set, it will use the directory where the docker-compose file is located.
-  #     dockerfile: ../backend/Dockerfile # This is the path to the dockerfile to use to build the service.
-  #     # We colocate the dockerfile with the code it is building, so we have to specify the path to it.
-  #     # on the other hand, we can bring the dockerfile to the same path as the docker-compose file, but then we have to specify the path to the code in the context directive.
-  # frontend:
-  #   ports:
-  #     - 3001:3000
-  #   environment:
-  #     - NEXT_PUBLIC_BACKEND_API_URL=http://backend:3000
-  #     - NODE_ENV=development
-  #   build:
-  #     context: ../frontend/
-  #     dockerfile: ../frontend/Dockerfile.dev
+  backend: # this is the backend service, which is the main service of the application
+    depends_on: # In defining depends_on, we define what we want to wait for before starting the service.
+      postgres:
+        condition: service_healthy # there are multiple conditions, but service_healthy is the last one to be checked, so it is the most reliable.
+    ports:
+      - 3000:3000
+    environment:
+      - DATABASE_URL=postgres://postgres:postgrespw@postgres:5432/postgres
+      - LOGGING_URL=http://loki:3100/loki/api/v1/push
+      - PROMETHEUS_URL=http://prometheus:9090
+      - REDIS_URL=redis://redis:6379
+      - STATSD_URL=statsdexporter:9125
+      - PORT=3000
+      - SAML_CERT_PATH=/certs/cert.pem
+      - SAML_KEY_PATH=/certs/key.pem
+      - IDP_URL=http://host.docker.internal:8080/idp.xml #for prod: https://idp.elon.edu/idp/shibboleth
+      - SP_URL=http://localhost:3000
+      - WEB_URL=http://localhost:3001
+    volumes:
+      - ./saml:/certs
+      - ./saml/cert.pem:/certs/cert.pem
+      - ./saml/key.pem:/certs/key.pem
+    build: # In build, we define where to build the provided docker file. This is in contrast to just supplying the image which is queried from a registry. This service is local and thus not avaiable in a registry, so we have to build it locally.
+      context: ../backend/ # Context describes where to build the dockerfile from. If not set, it will use the directory where the docker-compose file is located.
+      dockerfile: ../backend/Dockerfile # This is the path to the dockerfile to use to build the service.
+      # We colocate the dockerfile with the code it is building, so we have to specify the path to it.
+      # on the other hand, we can bring the dockerfile to the same path as the docker-compose file, but then we have to specify the path to the code in the context directive.
+  frontend:
+    ports:
+      - 3001:3000
+    environment:
+      - NEXT_PUBLIC_BACKEND_API_URL=http://backend:3000
+      - NODE_ENV=development
+    build:
+      context: ../frontend/
+      dockerfile: ../frontend/Dockerfile.dev
   mssql:
     image: mcr.microsoft.com/mssql/server:2022-latest
     environment:

simurgh/migrate.py

@@ -1,4 +1,5 @@
-# migrate.py is a
+# migrate.py is a script to fetch Elon's MSSQL datasets into our PSQL database
+# currenlty, this migrates Elon's Building dataset
 
 import pymssql as mssql
 import psycopg2 as pg
@@ -15,4 +16,4 @@ def main():
 
     b = a.fetchall()
 
-    print(b)
+    print(b)

simurgh/migrate_test.py

@@ -0,0 +1,76 @@
+# TODO: periodic update
+
+import pandas as pd
+from sqlalchemy import create_engine, exc, create_engine, exc, MetaData, Table, Column, VARCHAR, Numeric, TIMESTAMP, text
+
+# define connection variables
+mssql_user = "sa"
+mssql_pass = "Password123"
+mssql_host = "localhost:1433"
+mssql_database = "data"
+
+psql_user = "postgres"
+psql_pass = "postgrespw"
+psql_host = "localhost:5432"
+psql_database = "postgres"
+
+def main():
+    try:
+        mssql_engine = connect_mssql()
+        print("Connected to MSSQL successfully.")
+
+        psql_engine, buildings_table_psql = connect_psql()
+        print("Connected to PostgreSQL successfully.")
+
+        with psql_engine.connect() as psql_conn:
+                    # create PSQL buildings table, if doesn't exist
+                    buildings_table_psql.create(psql_engine)  
+                    # fetch mssql data
+                    mssql_data = pd.read_sql("SELECT * FROM dbo.buildings", mssql_engine)
+                    # insert into PSQL buildings table
+                    mssql_data.to_sql('buildings', psql_conn, index=False, if_exists='replace', method='multi')
+
+        print("Data copied from MSSQL to PostgreSQL.")
+
+    except exc.OperationalError as e:
+        print(f"Connection failed: {e}")
+
+
+def connect_mssql():
+    # define mssql connection object and engine
+    mssql_url_object = f"mssql+pyodbc://{mssql_user}:{mssql_pass}@{mssql_host}/{mssql_database}?driver=ODBC+Driver+17+for+SQL+Server"
+    mssql_engine = create_engine(mssql_url_object)
+    mssql_engine.connect()
+
+    return mssql_engine
+
+
+def connect_psql():
+    # define psql connection object and engine
+    psql_url_object = f"postgresql://{psql_user}:{psql_pass}@{psql_host}/{psql_database}"
+    psql_engine = create_engine(psql_url_object)
+    # define PSQL table structure
+    buildings_table_psql = Table(
+        'buildings', MetaData(),
+        Column('BUILDINGS_ID2', VARCHAR(4), primary_key=True, nullable=False),
+        Column('BLDG_DESC', VARCHAR(50)),
+        Column('BLDG_LOCATION', VARCHAR(5)),
+        Column('BLDG_LOCATION_REPRESENTATION', VARCHAR(30)),
+        Column('BLDG_TYPE', VARCHAR(10)),
+        Column('BLDG_TYPE_REPRESENTATION', VARCHAR(32)),
+        Column('BLDG_LONG_DESC', VARCHAR(1996)),
+        Column('BLDG_CITY', VARCHAR(25)),
+        Column('BLDG_STATE', VARCHAR(2)),
+        Column('BLDG_ZIP', VARCHAR(10)),
+        Column('BLDG_SECTOR', VARCHAR(10)),
+        Column('BLDG_SECTOR_REPRESENTATION', VARCHAR(32)),
+        Column('BLDG_LATITUDE', Numeric),
+        Column('BLDG_LONGITUDE', Numeric),
+        Column('BUILDINGS_ADD_DATE', TIMESTAMP),
+        Column('BUILDINGS_CHGDATE', TIMESTAMP)
+    )
+    return psql_engine, buildings_table_psql
+
+if __name__ == '__main__':
+     main()
+

simurgh/requirements.txt

@@ -1,2 +1,4 @@
 pymssql
 psycopg2-binary
+pandas
+sqlalchemy