diff --git a/infra/docker-compose.yml b/infra/docker-compose.yml index 9ea7dee..1f78d4e 100644 --- a/infra/docker-compose.yml +++ b/infra/docker-compose.yml @@ -79,42 +79,42 @@ services: # this is where services are defined. A service is a docker container, retries: 5 redis: image: redis:7.0.9 - # backend: # this is the backend service, which is the main service of the application - # depends_on: # In defining depends_on, we define what we want to wait for before starting the service. - # postgres: - # condition: service_healthy # there are multiple conditions, but service_healthy is the last one to be checked, so it is the most reliable. - # ports: - # - 3000:3000 - # environment: - # - DATABASE_URL=postgres://postgres:postgrespw@postgres:5432/postgres - # - LOGGING_URL=http://loki:3100/loki/api/v1/push - # - PROMETHEUS_URL=http://prometheus:9090 - # - REDIS_URL=redis://redis:6379 - # - STATSD_URL=statsdexporter:9125 - # - PORT=3000 - # - SAML_CERT_PATH=/certs/cert.pem - # - SAML_KEY_PATH=/certs/key.pem - # - IDP_URL=http://host.docker.internal:8080/idp.xml #for prod: https://idp.elon.edu/idp/shibboleth - # - SP_URL=http://localhost:3000 - # - WEB_URL=http://localhost:3001 - # volumes: - # - ./saml:/certs - # - ./saml/cert.pem:/certs/cert.pem - # - ./saml/key.pem:/certs/key.pem - # build: # In build, we define where to build the provided docker file. This is in contrast to just supplying the image which is queried from a registry. This service is local and thus not avaiable in a registry, so we have to build it locally. - # context: ../backend/ # Context describes where to build the dockerfile from. If not set, it will use the directory where the docker-compose file is located. - # dockerfile: ../backend/Dockerfile # This is the path to the dockerfile to use to build the service. - # # We colocate the dockerfile with the code it is building, so we have to specify the path to it. - # # on the other hand, we can bring the dockerfile to the same path as the docker-compose file, but then we have to specify the path to the code in the context directive. - # frontend: - # ports: - # - 3001:3000 - # environment: - # - NEXT_PUBLIC_BACKEND_API_URL=http://backend:3000 - # - NODE_ENV=development - # build: - # context: ../frontend/ - # dockerfile: ../frontend/Dockerfile.dev + backend: # this is the backend service, which is the main service of the application + depends_on: # In defining depends_on, we define what we want to wait for before starting the service. + postgres: + condition: service_healthy # there are multiple conditions, but service_healthy is the last one to be checked, so it is the most reliable. + ports: + - 3000:3000 + environment: + - DATABASE_URL=postgres://postgres:postgrespw@postgres:5432/postgres + - LOGGING_URL=http://loki:3100/loki/api/v1/push + - PROMETHEUS_URL=http://prometheus:9090 + - REDIS_URL=redis://redis:6379 + - STATSD_URL=statsdexporter:9125 + - PORT=3000 + - SAML_CERT_PATH=/certs/cert.pem + - SAML_KEY_PATH=/certs/key.pem + - IDP_URL=http://host.docker.internal:8080/idp.xml #for prod: https://idp.elon.edu/idp/shibboleth + - SP_URL=http://localhost:3000 + - WEB_URL=http://localhost:3001 + volumes: + - ./saml:/certs + - ./saml/cert.pem:/certs/cert.pem + - ./saml/key.pem:/certs/key.pem + build: # In build, we define where to build the provided docker file. This is in contrast to just supplying the image which is queried from a registry. This service is local and thus not avaiable in a registry, so we have to build it locally. + context: ../backend/ # Context describes where to build the dockerfile from. If not set, it will use the directory where the docker-compose file is located. + dockerfile: ../backend/Dockerfile # This is the path to the dockerfile to use to build the service. + # We colocate the dockerfile with the code it is building, so we have to specify the path to it. + # on the other hand, we can bring the dockerfile to the same path as the docker-compose file, but then we have to specify the path to the code in the context directive. + frontend: + ports: + - 3001:3000 + environment: + - NEXT_PUBLIC_BACKEND_API_URL=http://backend:3000 + - NODE_ENV=development + build: + context: ../frontend/ + dockerfile: ../frontend/Dockerfile.dev mssql: image: mcr.microsoft.com/mssql/server:2022-latest environment: diff --git a/simurgh/migrate.py b/simurgh/migrate.py index 659d431..cf62130 100644 --- a/simurgh/migrate.py +++ b/simurgh/migrate.py @@ -1,4 +1,5 @@ -# migrate.py is a +# migrate.py is a script to fetch Elon's MSSQL datasets into our PSQL database +# currenlty, this migrates Elon's Building dataset import pymssql as mssql import psycopg2 as pg @@ -15,4 +16,4 @@ def main(): b = a.fetchall() - print(b) + print(b) \ No newline at end of file diff --git a/simurgh/migrate_test.py b/simurgh/migrate_test.py new file mode 100644 index 0000000..91c042a --- /dev/null +++ b/simurgh/migrate_test.py @@ -0,0 +1,76 @@ +# TODO: periodic update + +import pandas as pd +from sqlalchemy import create_engine, exc, create_engine, exc, MetaData, Table, Column, VARCHAR, Numeric, TIMESTAMP, text + +# define connection variables +mssql_user = "sa" +mssql_pass = "Password123" +mssql_host = "localhost:1433" +mssql_database = "data" + +psql_user = "postgres" +psql_pass = "postgrespw" +psql_host = "localhost:5432" +psql_database = "postgres" + +def main(): + try: + mssql_engine = connect_mssql() + print("Connected to MSSQL successfully.") + + psql_engine, buildings_table_psql = connect_psql() + print("Connected to PostgreSQL successfully.") + + with psql_engine.connect() as psql_conn: + # create PSQL buildings table, if doesn't exist + buildings_table_psql.create(psql_engine) + # fetch mssql data + mssql_data = pd.read_sql("SELECT * FROM dbo.buildings", mssql_engine) + # insert into PSQL buildings table + mssql_data.to_sql('buildings', psql_conn, index=False, if_exists='replace', method='multi') + + print("Data copied from MSSQL to PostgreSQL.") + + except exc.OperationalError as e: + print(f"Connection failed: {e}") + + +def connect_mssql(): + # define mssql connection object and engine + mssql_url_object = f"mssql+pyodbc://{mssql_user}:{mssql_pass}@{mssql_host}/{mssql_database}?driver=ODBC+Driver+17+for+SQL+Server" + mssql_engine = create_engine(mssql_url_object) + mssql_engine.connect() + + return mssql_engine + + +def connect_psql(): + # define psql connection object and engine + psql_url_object = f"postgresql://{psql_user}:{psql_pass}@{psql_host}/{psql_database}" + psql_engine = create_engine(psql_url_object) + # define PSQL table structure + buildings_table_psql = Table( + 'buildings', MetaData(), + Column('BUILDINGS_ID2', VARCHAR(4), primary_key=True, nullable=False), + Column('BLDG_DESC', VARCHAR(50)), + Column('BLDG_LOCATION', VARCHAR(5)), + Column('BLDG_LOCATION_REPRESENTATION', VARCHAR(30)), + Column('BLDG_TYPE', VARCHAR(10)), + Column('BLDG_TYPE_REPRESENTATION', VARCHAR(32)), + Column('BLDG_LONG_DESC', VARCHAR(1996)), + Column('BLDG_CITY', VARCHAR(25)), + Column('BLDG_STATE', VARCHAR(2)), + Column('BLDG_ZIP', VARCHAR(10)), + Column('BLDG_SECTOR', VARCHAR(10)), + Column('BLDG_SECTOR_REPRESENTATION', VARCHAR(32)), + Column('BLDG_LATITUDE', Numeric), + Column('BLDG_LONGITUDE', Numeric), + Column('BUILDINGS_ADD_DATE', TIMESTAMP), + Column('BUILDINGS_CHGDATE', TIMESTAMP) + ) + return psql_engine, buildings_table_psql + +if __name__ == '__main__': + main() + diff --git a/simurgh/requirements.txt b/simurgh/requirements.txt index 0460126..f3c689b 100644 --- a/simurgh/requirements.txt +++ b/simurgh/requirements.txt @@ -1,2 +1,4 @@ pymssql psycopg2-binary +pandas +sqlalchemy \ No newline at end of file