import requests

# use the get function
page = requests.get("https://www.w3schools.com/howto/tryhow_make_a_website.htm")

from bs4 import BeautifulSoup as bs 

import csv

# we specify the url that we are interested in
url = "https://ideas.repec.org"

# we use the url to get the page that we are interested in
page = requests.get(url+"/i/ex.html").text

# we use the Beautiful Soup library to on this page
soup = bs(page)

# some trial and error:
soup.body
# print(soup.body.prettify())
soup.body
soup.body.a
soup.body.td
soup.body.tr
soup.body.table

soup.body.table.a # this looks like what we need
person = soup.body.table.a

# take the name
name = person.string # we get the string
lastName, firstName = name.split(",") # we split first and last name
lastName = lastName.strip() # we get rid of spaces at beginning and end
firstName = firstName.strip()

# take the link for the personal info
link = person["href"]
    
# take the number of papers 
# we could have used regular expressions for this
papers = person.parent.text.split("(")[1]
papers = papers[:papers.find(")")] 

# puts all the a tags within the table in a list
people = soup.body.table.find_all("a")
# initialise the list where all the data will go
data = []
for person in people:
    # take the name
    name = person.string # we get the string
    lastName, firstName = name.split(",") # we split first and last name
    lastName = lastName.strip() # we get rid of spaces at beginning and end
    firstName = firstName.strip()

    # take the link for the personal info
    link = person["href"]

    # take the number of papers 
    # we could have used regular expressions for this
    papers = person.parent.text.split("(")[1]
    papers = papers[:papers.find(")")] 
    
    # append the data to our list
    data.append((firstName, lastName, link, papers))

# we want to avoid downloading all the avialable data because 
# we do not want to disturb the servers. Also data on 10 individuals 
# is enough for our purposes
finalData=[] # this list will store the final data that we want
for personInfo in data[:10]:
    # now for each person we load a different page
    personURL = personInfo[2]
    personPage = requests.get(url+personURL).text
    # we make a soup for each person
    personSoup = bs(personPage)
    
    # we navigate to the information that we want and we pick
    # what we are interested in
    affiliation = personSoup.find(id="affiliation").h3.contents
    # the following line we do not really need but it is a way of getting rid of the 
    # elements of the list that are not proper elements (this works for this case)
    affiliation = [element.string for element in affiliation if element.string!=None]
    
    # append to our list of data
    finalData.append((personInfo[0], personInfo[1], personInfo[3], affiliation[-1]))
    

with open("data.csv", "w", newline="") as csvFile:
    writer = csv.writer(csvFile, delimiter= ";")
    writer.writerow(["first name", "last name", "number of papers", "affiliation"])
    for row in finalData:
        writer.writerow(row)

import pandas as pd

# save the table
rows = soup.body.table.find_all("tr")

# initialise data
frameData = []

for row in rows: # go through all the rows
    cells = row.find_all("td") # cell in each row (one for each column)
    # temp = [cell.string() for cell in cells] # string gives none
    try:
        temp = [cell.a.get_text().strip() for cell in cells]
        frameData.append(temp)
    except:
        pass
    
df = pd.DataFrame(frameData)
# here we just have a list of names so the pandas data frame is not that useful
# in other cases the different columns of the table may contain different variables,
# these names you could also extract from the table and then put them in DataFrame()
# then you can add the names with pandas so that you can later reference them easier:
# df = pd.DataFrame(frameData, columns=some_list_of_names)