Big-Data-Analytics/f_test.R at master · nomanalahi/Big-Data-Analytics · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
library(tidyverse)
library(dplyr)
library(caret)
library(rpart)
library(rpart.plot)
library(caTools)
library(tree)
library(hms)
library(party)
library(gbm)
library(ggplot2)
library(plyr)

#reading the flights dataset
dataset <- read.csv("flights.csv")
airline <- read.csv("airlines.csv")
airport <- read.csv("airports.csv")

#deducting all rows which have the records of cancellation and diversion
dataset <- dataset %>% filter(CANCELLED == 0) %>% filter(DIVERTED == 0)

#adding new column of delay result
dataset <- mutate(dataset, DELAY_RESULT = ARRIVAL_DELAY)

#categorizing delay result from numeric to yes or no
dataset$DELAY_RESULT <-ifelse(dataset$DELAY_RESULT<15, "No", "Yes")

#converting delay result to factor
dataset$DELAY_RESULT <- as.factor(dataset$DELAY_RESULT)

#explore dataset
head(dataset$DELAY_RESULT)
names(dataset)
class(dataset$DELAY_RESULT)
typeof(dataset$DELAY_RESULT)
prop.table(table(dataset$DELAY_RESULT))
nrow(dataset)


#selecting necessary features only
dataset <- dataset %>% select("MONTH", "AIRLINE", "DAY_OF_WEEK", "FLIGHT_NUMBER",
            "ORIGIN_AIRPORT", "DESTINATION_AIRPORT", "SCHEDULED_DEPARTURE", "DELAY_RESULT")

#checking dataset
head(dataset)
head(airline)
head(airport)

#changing column airline's value
colnames(dataset)[colnames(dataset) == "AIRLINE"] <- "IATA_CODE"
dataset <- merge(dataset, airline, by.x="IATA_CODE", by.y="IATA_CODE")
dataset <- dataset %>% select(- "IATA_CODE")


#changing column origin_airport's value
colnames(dataset)[colnames(dataset) == "ORIGIN_AIRPORT"] <- "IATA_CODE"
dataset <- merge(dataset, airport, by.x="IATA_CODE", by.y="IATA_CODE")
dataset <- dataset %>% select(- c("IATA_CODE", "CITY", "STATE", "COUNTRY", "LATITUDE", "LONGITUDE"))
colnames(dataset)[colnames(dataset) == "AIRPORT"] <- "ORIGIN_AIRPORT"


#changing column DESTINATION_AIRPORT's value
colnames(dataset)[colnames(dataset) == "DESTINATION_AIRPORT"] <- "IATA_CODE"
dataset <- merge(dataset, airport, by.x="IATA_CODE", by.y="IATA_CODE")
dataset <- dataset %>% select(- c("IATA_CODE", "CITY", "STATE", "COUNTRY", "LATITUDE", "LONGITUDE"))
colnames(dataset)[colnames(dataset) == "AIRPORT"] <- "DESTINATION_AIRPORT"


#EDA
counts <- table(dataset$AIRLINE)/1000
barplot(counts, main="Airline Distribution",
        ylab="Number of Filghts(in thousands)", las=2, cex.names=.5)


pal <- colorRampPalette(colors = c("lightblue", "blue"))(3)
counts <- table(dataset$MONTH)/1000
barplot(counts, main="No. of Flights(Monthly)", col = pal,
        ylab="Number of Filghts(in thousands)", las=1, cex.names=.5)


pal <- colorRampPalette(colors = c("lightgreen", "green"))(3)
counts <- table(dataset$DAY_OF_WEEK)/1000
barplot(counts, main="No. of Flights(Daily)", col = pal,ylim=c(0,800),
        ylab="Number of Filghts(in thousands)", las=1, cex.names=.5)


w <- table(dataset$DAY_OF_WEEK)
t = as.data.frame(w)
t$Freq <- t$Freq/5231130

pie = ggplot(t, aes(x="", y=Freq, fill=Var1)) + geom_bar(stat="identity", width=1)
pie = pie + coord_polar("y", start=0) + geom_text(aes(label = paste0(round(t$Freq*100), "%")), position = position_stack(vjust = 0.5))
pie = pie + labs(x = NULL, y = NULL, fill = NULL, title = "Percentage of Flights (Daily)")
pie = pie + theme_classic() + theme(axis.line = element_blank(),
                                    axis.text = element_blank(),
                                    axis.ticks = element_blank(),
                                    plot.title = element_text(hjust = 0.5, color = "#666666"))

pie


#find index to suffle all rows
shuffle_index <- sample(1:nrow(dataset))
head(shuffle_index)

#suffle all rows
dataset <- dataset[shuffle_index, ]
head(dataset)

#creating traing an testing dataset
training_index <- createDataPartition(dataset$DELAY_RESULT, p=0.80, list=FALSE)
training_data <- dataset[training_index,]
testing_data <- dataset[-training_index,]

head(training_data)
head(testing_data)

#probability of YES and NO
prop.table(table(dataset$DELAY_RESULT))


#tree
tree <- rpart(DELAY_RESULT~., data = training_data, method = 'class')
rpart.plot(tree, extra = 106)

predict_unseen <-predict(tree, testing_data, type = 'class')

table_mat <- table(testing_data$DELAY_RESULT, predict_unseen)
table_mat

accuracy_Test <- sum(diag(table_mat)) / sum(table_mat)
accuracy_Test


#boosting
# train GBM model
gbm.fit <- gbm(
  formula = DELAY_RESULT~.,
  distribution = "bernoulli",
  data = training_data,
)

print(gbm.fit)

sqrt(min(gbm.fit$cv.error))

gbm.perf(gbm.fit, method = "cv")

pred <- predict(gbm.fit, n.trees = gbm.fit$n.trees, testing_data)

training_data$DELAY_RESULT <- as.factor(training_data$DELAY_RESULT)

table_mat <- table(testing_data$DELAY_RESULT, pred)
table_mat

accuracy_Test <- sum(diag(table_mat)) / sum(table_mat)
accuracy_Test