From 686100ed91cee0314908954986c4afdedd36e6fa Mon Sep 17 00:00:00 2001 From: jayghoshkhemka Date: Thu, 28 Nov 2024 11:49:05 +0530 Subject: [PATCH 1/9] Add files via upload --- deplaoyment.yaml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 deplaoyment.yaml diff --git a/deplaoyment.yaml b/deplaoyment.yaml new file mode 100644 index 00000000..e022f0ff --- /dev/null +++ b/deplaoyment.yaml @@ -0,0 +1,26 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + app: vote + tier: front + name: vote +spec: + replicas: 2 + selector: + matchLabels: + app: vote + strategy: {} + template: + metadata: + creationTimestamp: null + labels: + app: vote + tier: front + spec: + containers: + - image: schoolofdevops/vote:v1 + name: vote + resources: {} +status: {} \ No newline at end of file From 8746d385db013c890e9996aa6e8c3dccc10b4b9f Mon Sep 17 00:00:00 2001 From: jayghoshkhemka Date: Thu, 28 Nov 2024 11:50:36 +0530 Subject: [PATCH 2/9] Add files via upload --- service.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 service.yaml diff --git a/service.yaml b/service.yaml new file mode 100644 index 00000000..694ff923 --- /dev/null +++ b/service.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + creationTimestamp: null + labels: + app: vote + tier: front + name: vote +spec: + ports: + - name: "80" + nodePort: 30000 + port: 80 + protocol: TCP + targetPort: 80 + selector: + app: vote + type: NodePort +status: + loadBalancer: {} \ No newline at end of file From a15aa3efd4e912340c60b22141ae7403ff0ed8fe Mon Sep 17 00:00:00 2001 From: jayghoshkhemka Date: Thu, 28 Nov 2024 11:55:49 +0530 Subject: [PATCH 3/9] /githubtraining From f4990f418c76180ffc201885075c228d4f9b138e Mon Sep 17 00:00:00 2001 From: jayghoshkhemka Date: Thu, 28 Nov 2024 12:14:50 +0530 Subject: [PATCH 4/9] Update deplaoyment.yaml --- deplaoyment.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/deplaoyment.yaml b/deplaoyment.yaml index e022f0ff..64350703 100644 --- a/deplaoyment.yaml +++ b/deplaoyment.yaml @@ -10,8 +10,6 @@ spec: replicas: 2 selector: matchLabels: - app: vote - strategy: {} template: metadata: creationTimestamp: null @@ -23,4 +21,4 @@ spec: - image: schoolofdevops/vote:v1 name: vote resources: {} -status: {} \ No newline at end of file +status: {} From 43f1a41d27e4418c8c588498c1fad66e60325d7c Mon Sep 17 00:00:00 2001 From: jayghoshkhemka Date: Thu, 28 Nov 2024 13:00:08 +0530 Subject: [PATCH 5/9] Update deployment.yaml --- deplaoyment.yaml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/deplaoyment.yaml b/deplaoyment.yaml index 64350703..93125cb0 100644 --- a/deplaoyment.yaml +++ b/deplaoyment.yaml @@ -9,16 +9,3 @@ metadata: spec: replicas: 2 selector: - matchLabels: - template: - metadata: - creationTimestamp: null - labels: - app: vote - tier: front - spec: - containers: - - image: schoolofdevops/vote:v1 - name: vote - resources: {} -status: {} From 6a8ded9f2988fb51d12ba279257e092414bd28d7 Mon Sep 17 00:00:00 2001 From: jayghoshkhemka Date: Fri, 29 Nov 2024 05:41:11 +0000 Subject: [PATCH 6/9] q q --- Mixed_Language_Demo.py | 80 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 Mixed_Language_Demo.py diff --git a/Mixed_Language_Demo.py b/Mixed_Language_Demo.py new file mode 100644 index 00000000..87dbfaff --- /dev/null +++ b/Mixed_Language_Demo.py @@ -0,0 +1,80 @@ +# Databricks notebook source +# MAGIC %md +# MAGIC # Simple Databricks Notebook with Python, R, SQL, and Markdown +# MAGIC This notebook demonstrates a combination of Python, R, SQL, and Markdown cells +# MAGIC in Databricks. It performs basic operations, suitable for a low-cost AWS +# MAGIC Databricks environment. +# MAGIC ### Steps: +# MAGIC 1. Perform basic Python operations. +# MAGIC 2. Use R for basic calculations. +# MAGIC 3. Execute SQL queries on an in-memory table. +# MAGIC 4. Add comments and explanations using Markdown. + +# COMMAND ---------- + +# Python: Create a sample data frame and display it +import pandas as pd +# Sample data +data = {"Name": ["Alice", "Bob", "Charlie"], "Age": [25, 30, 35], "Salary": +[50000, 60000, 70000]} +df = pd.DataFrame(data) +# Convert to Spark DataFrame +spark_df = spark.createDataFrame(df) +# Display the Spark DataFrame +display(spark_df) + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ### Python Code Explanation: +# MAGIC - Created a Pandas DataFrame with sample data (Name, Age, Salary). +# MAGIC - Converted it into a Spark DataFrame for use in Databricks. +# MAGIC +# MAGIC - Displayed the Spark DataFrame using the `display` function. + +# COMMAND ---------- + +# MAGIC %r +# MAGIC R: Perform basic operations +# MAGIC # Load a small vector of numbers and calculate their mean +# MAGIC numbers <- c(10, 20, 30, 40, 50) +# MAGIC mean_value <- mean(numbers) +# MAGIC # Print the mean value +# MAGIC print(paste("The mean of the numbers is:", mean_value)) + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ### R Code Explanation: +# MAGIC - Created a numeric vector `numbers` with five values. +# MAGIC - Calculated the mean of the numbers using the `mean` function. +# MAGIC - Printed the result to the output. + +# COMMAND ---------- + +# MAGIC %sql +# MAGIC +# MAGIC -- SQL: Create and query a temporary table +# MAGIC CREATE OR REPLACE TEMP VIEW employee AS +# MAGIC SELECT * FROM VALUES +# MAGIC ('Alice', 25, 50000), +# MAGIC ('Bob', 30, 60000), +# MAGIC ('Charlie', 35, 70000) +# MAGIC AS employees(Name, Age, Salary); +# MAGIC -- Query the table +# MAGIC SELECT Name, Salary FROM employee WHERE Salary > 55000; + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ### SQL Code Explanation: +# MAGIC - Created a temporary table `employee` using the `CREATE OR REPLACE TEMP VIEW` +# MAGIC command. +# MAGIC - Inserted sample data for employees (Name, Age, Salary). +# MAGIC - Queried the table to select employees with a salary greater than 55,000. + +# COMMAND ---------- + +# Python: Stop all active sessions to minimize costs +# This ensures you don't leave any expensive resources running +spark.catalog.clearCache() From 022e63f3f593394a03f4af23ca7022e4266ad219 Mon Sep 17 00:00:00 2001 From: jayghoshkhemka Date: Fri, 29 Nov 2024 12:58:42 +0530 Subject: [PATCH 7/9] Create test_notebook.py --- tests/test_notebook.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 tests/test_notebook.py diff --git a/tests/test_notebook.py b/tests/test_notebook.py new file mode 100644 index 00000000..14aa49d4 --- /dev/null +++ b/tests/test_notebook.py @@ -0,0 +1,25 @@ +import pytest +import Mixed_Language_Demo as databrics_code # Import the module +def test_dataframe_creation(): +""" +Test for the `create_dataframe` function +""" +# Create the DataFrame +df = databrics_code.create_dataframe() +# Verify the DataFrame structure +assert df is not None +assert len(df) == 3 # Three rows +assert "Name" in df.columns + +assert list(df["Name"]) == ["Amar", "Akbar", "Anthony"] +def test_dataframe_summary(): +""" +Test the `print_dataframe_summary` function +""" +# Create the DataFrame +df = databrics_code.create_dataframe() +# Ensure no exceptions are raised during summary printing +try: +databrics_code.print_dataframe_summary(df) +except Exception as e: +pytest.fail(f"Summary printing failed with error: {e}") From 7e3727b3c898306d9494b99d61aaed7c62206011 Mon Sep 17 00:00:00 2001 From: jayghoshkhemka Date: Fri, 29 Nov 2024 13:01:19 +0530 Subject: [PATCH 8/9] Create requirements.txt --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..7139071d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +pandas +pytest From b401276beabd4882110aae7ad375ac6e27b79eac Mon Sep 17 00:00:00 2001 From: jayghoshkhemka Date: Fri, 29 Nov 2024 13:02:30 +0530 Subject: [PATCH 9/9] Create test-notebook.yml --- .github/workflows/test-notebook.yml | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/test-notebook.yml diff --git a/.github/workflows/test-notebook.yml b/.github/workflows/test-notebook.yml new file mode 100644 index 00000000..2f9d96a3 --- /dev/null +++ b/.github/workflows/test-notebook.yml @@ -0,0 +1,30 @@ +name: Test Notebook Code +# Trigger the workflow on push events to all branches +on: +push: + +branches: +- '*' +jobs: +test: +runs-on: ubuntu-latest +steps: +- name: Set PYTHONPATH +run: echo "PYTHONPATH=$PYTHONPATH:$(pwd)" >> $GITHUB_ENV +# Step 1: Checkout the repository +- name: Checkout code +uses: actions/checkout@v3 +# Step 2: Set up Python environment +- name: Set up Python +uses: actions/setup-python@v4 +with: +python-version: 3.9 +# Step 3: Install dependencies +- name: Install dependencies +run: | +pip install -r requirements.txt +pip install pytest +# Step 4: Run tests +- name: Run tests +run: | +pytest tests/