From f20cf6af72e89d0ceef2307c57dcea2d85ee730a Mon Sep 17 00:00:00 2001 From: Luiza Andrade Date: Thu, 20 Apr 2023 19:20:50 -0500 Subject: [PATCH 1/2] Create list-villages.py --- code/jjm-dashboard-scraping/list-villages.py | 36 ++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 code/jjm-dashboard-scraping/list-villages.py diff --git a/code/jjm-dashboard-scraping/list-villages.py b/code/jjm-dashboard-scraping/list-villages.py new file mode 100644 index 0000000..ce4ff51 --- /dev/null +++ b/code/jjm-dashboard-scraping/list-villages.py @@ -0,0 +1,36 @@ +import requests +import pandas as pd +import os + +#create StCode list +stcode_list = ['461','231','211', ] +#create DtCode list +dtcode_list = ['4951','4881','4891','4921','4941','4991','4931','4%3A61','48%3A1','4971','4%3A%3A1','4%3A81','4%3A51','4%3A51','4%3A71','4%3A31','4%3A41','4811','4%3A21','4841','49%3A1','4911','4821','4%3A11','4861','4%3A91','4871','4961','4981','4831','4851'] +#create alphabet_search list +alphabetsearch_lst = ['B1','C1','D1','E1','F1','G1','H1','I1','J1','K1','L1','M1','N1','O1','P1','Q1','R1','S1','T1','U1','V1','W1','X1','Y1','Z1','%5B1'] +#create empty dataframe +df=pd.DataFrame() +#Define session +s=requests.session() +#Define API url +url_search='https://ejalshakti.gov.in/jjmreport/JJMIndia.aspx/Bind_search_Village' +#Define header for the post request +headers={'user-agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36'} +#Define payload for the request form +data={ + 'DtCode': "4951", + 'StCode': "321", + 'VillageName': "B1" + } +req=s.post(url_search,headers=headers,json=data) + + +#getdataVillageClick('21','384','404976','0000305249','Odisha','Angul','Abmadara') +#read the data back as json file +j=req.json() +#convert the json file to dataframe +df_new=pd.DataFrame(j['d']) +#append dataframe to other dataframe +df = pd.concat([df,df_new]) +#save the dataframe as csv file +df.to_csv('../../data/scraping/village-list.csv') From cba7dfee307ea1c3fa4468c21196a9951e1f5bdb Mon Sep 17 00:00:00 2001 From: Luiza Andrade Date: Tue, 25 Apr 2023 12:31:50 -0500 Subject: [PATCH 2/2] Add more villages to Odisha data --- .../get-village-info.py | 83 +++++++++++++++++++ code/jjm-dashboard-scraping/list-villages.py | 79 +++++++++++------- 2 files changed, 131 insertions(+), 31 deletions(-) create mode 100644 code/jjm-dashboard-scraping/get-village-info.py diff --git a/code/jjm-dashboard-scraping/get-village-info.py b/code/jjm-dashboard-scraping/get-village-info.py new file mode 100644 index 0000000..fc638be --- /dev/null +++ b/code/jjm-dashboard-scraping/get-village-info.py @@ -0,0 +1,83 @@ +import requests +import pandas as pd +import os +from glob import glob + +df = pd.DataFrame() +s = requests.session() + +url_search='https://ejalshakti.gov.in/jjmreport/JJMVillage_Profile.aspx/Bind_Fhtc_info' +headers={'user-agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36'} + +old_data = pd.read_csv('../../data/scraping/village-data-odisha.csv') +villagesCollected = old_data['VillCode'] + +search_parameters = pd.read_csv('../../data/scraping/odisha-first-villcode.csv') +search_parameters = search_parameters[search_parameters['Collected'] == 0] +search_parameters = search_parameters.reset_index() + +for index, row in search_parameters.iterrows(): + DtCode = row['DtCode'] + StCode = row['StCode'] + VillCode = row['VillCode'] + i = 0 + while i < 15: + print(VillCode) + payloadStCode = str(StCode) + payloadStCode = payloadStCode.replace('0', '%3A') + payloadDtCode = str(DtCode) + payloadDtCode = payloadDtCode.replace('0', '%3A') + payloadVillCode = str(VillCode) + payloadVillCode = payloadVillCode.replace('0', '%3A') + + if not payloadVillCode in villagesCollected: + payload = { + "Cat" : "11", + "DtCode11" : payloadDtCode, # This also varies a bit + "Param" : "11", + "StCode11" : payloadStCode, + "SubCat" : "11", + "VillCode" : payloadVillCode + } + req = s.post(url_search, headers = headers, json = payload) + info = req.json() + df_new = pd.DataFrame(info['d']) + if not df_new.empty: + df_new["VillCode"] = payloadVillCode + df_new['DtCode'] = payloadDtCode + df_new['StCode'] = StCode + print('clean data') + df = pd.concat([df, df_new]) + df = df.loc[:, ~df.columns.str.contains('^Unnamed')] + i = 0 + if df_new.empty: + i = i + 1 + print(i) + VillCode = VillCode + 10 + if not df.empty: + file = '../../data/scraping/odisha/village-data-' + payloadDtCode + '.csv' + if os.path.exists(file): + print('combining with old data') + df_old = pd.read_csv(file, low_memory=False) + df_old = df_old.drop_duplicates() + df = pd.concat([df, df_old]) + df = df.drop_duplicates() + df.to_csv(file) + + +files = glob(os.path.join('../../data/scraping/odisha', '*.csv')) + +df = pd.DataFrame() +for file in files: + print(file) + df_i = pd.read_csv(file) + # Assumes everyone has same columns + df = pd.concat((df, df_i)) + +df = df.loc[:, ~df.columns.str.contains('^Unnamed')] +df = df.drop_duplicates() + +village_names = pd.read_csv('../../data/scraping/village-list-odisha.csv') +df = village_names.merge(df, how = 'left', on = 'VillageId') + +df.to_csv('../../data/scraping/village-data-odisha.csv') diff --git a/code/jjm-dashboard-scraping/list-villages.py b/code/jjm-dashboard-scraping/list-villages.py index ce4ff51..7b0e41d 100644 --- a/code/jjm-dashboard-scraping/list-villages.py +++ b/code/jjm-dashboard-scraping/list-villages.py @@ -1,36 +1,53 @@ import requests import pandas as pd import os +import json +from random import randint +from time import sleep -#create StCode list -stcode_list = ['461','231','211', ] -#create DtCode list -dtcode_list = ['4951','4881','4891','4921','4941','4991','4931','4%3A61','48%3A1','4971','4%3A%3A1','4%3A81','4%3A51','4%3A51','4%3A71','4%3A31','4%3A41','4811','4%3A21','4841','49%3A1','4911','4821','4%3A11','4861','4%3A91','4871','4961','4981','4831','4851'] -#create alphabet_search list +s = requests.session() +headers = {'user-agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36'} + +# List states +url = " https://ejalshakti.gov.in/jjmreport/JJMIndia.aspx/JJM_StateDistrictSearch" +payload = { + "StCode" : "11", + "Name" : "1" +} + +req = s.post(url, headers = headers, json = payload) +info = req.json() +states_df = pd.DataFrame(info['d']) + +# Dont overwrite: was manually edited to inlcude the 'StCode' column +# states_df.to_csv('../../data/scraping/states-list.csv') + +req = s.post(url, headers = headers, json = payload) +info = req.json() + +dtcode_list = ['4951','4881', '4891','4921','4941','4991','4931','4%3A61','48%3A1','4971','4%3A%3A1','4%3A81','4%3A51','4%3A71','4%3A31','4%3A41','4811','4%3A21','4841','49%3A1','4911','4821','4%3A11','4861','4%3A91','4871','4961','4981','4831','4851'] alphabetsearch_lst = ['B1','C1','D1','E1','F1','G1','H1','I1','J1','K1','L1','M1','N1','O1','P1','Q1','R1','S1','T1','U1','V1','W1','X1','Y1','Z1','%5B1'] -#create empty dataframe -df=pd.DataFrame() -#Define session -s=requests.session() -#Define API url -url_search='https://ejalshakti.gov.in/jjmreport/JJMIndia.aspx/Bind_search_Village' -#Define header for the post request -headers={'user-agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36'} -#Define payload for the request form -data={ - 'DtCode': "4951", - 'StCode': "321", - 'VillageName': "B1" - } -req=s.post(url_search,headers=headers,json=data) - - -#getdataVillageClick('21','384','404976','0000305249','Odisha','Angul','Abmadara') -#read the data back as json file -j=req.json() -#convert the json file to dataframe -df_new=pd.DataFrame(j['d']) -#append dataframe to other dataframe -df = pd.concat([df,df_new]) -#save the dataframe as csv file -df.to_csv('../../data/scraping/village-list.csv') +villages_df = pd.DataFrame() +url = 'https://ejalshakti.gov.in/jjmreport/JJMIndia.aspx/Bind_search_Village' + +#for StCode in "321": +for DtCode in dtcode_list: + print(DtCode) + for letter in alphabetsearch_lst: + print(letter) + payload = { + 'DtCode': DtCode, + 'StCode': '321', + 'VillageName': letter + } + req = s.post(url, headers = headers, json = payload) + info = req.json() + new_village = pd.DataFrame(info['d']) + if not new_village.empty: + new_village['DtCode'] = DtCode + new_village['StCode'] = '321' + villages_df = pd.concat([villages_df, new_village]) + +villages_df = villages_df.drop_duplicates() +villages_df.to_csv('../../data/scraping/village-list-odisha.csv') +