Sentimine/OtherTesting.R at master · robhurry/Sentimine · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
search.term <- "Bitcoin"
dateRef<-as.Date("2017-09-02")
daysBack<-1
url<-getBCPricesURL(dateStart=dateRef, daysBack=daysBack)
url
dataOut<-fromJSON(url)
dataOut$bpi


#Let's have a look at the rankings
head(title_data)
head(descrip_data)


head(sentimentCollection)
rob<-unlist(sentimentCollection)
remove99<-c(99)
sentimentCollectionClean<-rob [! rob %in% remove99]
summary(sentimentCollectionClean)

sesh<-html_session("http://www.google.com")

search <- html_form(read_html("http://www.google.com"))[[1]]
set_values(search, q = "bitcoin")
set_values(search, num = "100")
set_values(search, num = "100")
set_values(search, tbs = "cdr:1,cd_min:12/1/2017,cd_max:12/9/2017")
set_values(search, tbm = "nws")


values<-submit_form(sesh, search, submit = "btnG")
html_nodes(read_html(values), '.st')
values
values
## Not run: set_values(search, btnI = "blah")

library(twitteR)
#setup_twitter_oauth(consumer_key, consumer_secret, access_token=NULL, access_secret=NULL)
setup_twitter_oauth("RqxwQvin3yeZrQfCrrRA", "MimaKfV5IQQdCHabzxkZXan2f0W5W79IZsLPiusbfs", access_token="210046899-AQfbSWGL6hMkClufkFOadPJJLbJ0vy4F2zswyqf9", access_secret="wqUDuMl552R6EYTLjVyEHrtkTS4l0BZdIvtXLil832Cuq")
searchTwitter(searchString="bitcoin", since="2017-09-01", until="2017-09-07")

getTwitterURL <- function(search.term, dateRef, daysBack, domain = '.com', quotes=TRUE)
{

  dateEnd<- format(as.Date(dateRef), format="%Y-%m-%d")
  dateStart<- format(as.Date(dateRef)-daysBack, format="%Y-%m-%d")

  search.term <- gsub(' ', '%20', search.term)
  if(quotes) search.term <- paste('%22', search.term, '%22', sep='')
  getTwitterURL <- paste('https://twitter', domain, '/search?f=tweets&q=',
                        search.term,
                        'since%3A',dateStart,'%20%20until%3A',dateEnd,
                        '&src=typd'
                        , sep='')
}
library(httr)
library('rvest')
url<-getTwitterURL("bitcoin","2017-09-10", 5)
#webpage <- read_html(url)
response <- GET(url, add_headers('user-agent' = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36'))
webpage <- content(response)
#Using CSS selectors to scrape the descrip
descrip_data_html <- html_nodes(webpage,'.tweet-text')
#Converting the descrip data to text
descrip_data <- html_text(descrip_data_html)
descrip_data
webpage