coverter.py

from numpy.core.fromnumeric import shape
import pandas as pd
import random

#declaring random generator

# Read excel sheet generated by crawler
df = pd.read_csv(r'D:/USER FILES/DESKTOP/SASTRA/CSE SEM -7/BDA/mini proj/sitegraph/sitegraph/sitegraph.csv')
temp=[]

# Separage the one to many mapped links to one to one to save edges
no_of_edges=0
for label,row in df.iterrows():
    if isinstance(row['linkedurls'],str) and isinstance(row['url'],str) :
        urls=row['linkedurls'][1:-1].split(', ')
        for links in urls:
            links=links[2:-1]
            temp.append([row['url'],links])

# To generate range of values from 0 to maxRange
def Generator(maxRange):
  return [item for item in range(maxRange)]

maxRange=27599
n=int(10*27599/100)
TXT = random.sample(Generator(maxRange),n)
TXT.sort()

#Save it in topic file for use of Topic-sensitive algorithm
with open("Topics.txt",'w') as output:
  for row in TXT:
    output.write(str(row)+'\n')

# Assign node no. as integer for every node. 
idx=0
seen_first={}
final_data=[]
for x in temp:
    for links in x:
        if links not in seen_first:
            seen_first[links]=idx
            idx += 1
    data=[seen_first[links] for links in x]
    #if idx>=10: break
    #if data[0] != data[1]:
    final_data.append(data)

# Gnerate csv file that contains "URL - NODE number" for all nodes
(pd.DataFrame.from_dict(data=seen_first, orient='index')
    .to_csv('nodedata.csv',header=False))
#print(seen_first)

# Generate csv file and text file for edgesss
df_final = pd.DataFrame(final_data,columns=['url','linkedurl'])
df_final = df_final.drop_duplicates()
#print(df_final)
df_final = df_final.sort_values(['url','linkedurl'],ascending=[True,True])
print("No of Nodes = "+str(idx+1)+"\nNo of Edges = "+str(df_final.shape[0]))
df_final.to_csv("mylist.csv",index=False,header=['Source','Target'])
df_final.to_csv("mylist.txt",index=False,header=False,sep='\t')
print("Files created Successfully!!!")