import requests
from bs4 import BeautifulSoup
import json
import csv

counter = 1
course_list = list()
title_list = list()
teacher_list = list()
price_list = list()
duration_list = list()

for page_id in range(1, 3):
    url = f'https://toplearn.com/courses?pageId={page_id}'
    response = requests.get(url)

    html = response.text
    soup = BeautifulSoup(html, "html.parser")

    for h2s in soup.find_all('h2'):
        for a_tag in h2s.find('a'):
            title_list.append(a_tag.text)
            # print(f'course {a_tag.text} is added')
            
    for div_top in soup.find_all('div', class_='top'):
        for a_tag_teacher in div_top.find('a'):
            teacher_list.append(a_tag_teacher.get_text(strip=True))
            
            
            
            
    for span_time in soup.find_all('span', class_='time'):
        duration_list.append(span_time.get_text(strip=True))
       

    for span_price in soup.find_all('span', class_='price'):
        for i_tag in span_price.find('i'):
            if i_tag == 'رایگانـ':
                price_list.append(0)
            else:
                price_list.append(i_tag.get_text(strip=True))
            
    print(duration_list, len(duration_list))        
    for index, title in enumerate(title_list):
        course_list.append({
            'id': f'{page_id}-{index}', 
            'title': title,
            'teacher': teacher_list[2*index+1],
            'price': price_list[index],
            'duration': duration_list[index]
        })
                      
with open('courses.json', 'w', encoding='utf-8') as f:
    json.dump(course_list, f, ensure_ascii=False)
        


keys = course_list[0].keys()



with open('courses.csv', 'w', newline='', encoding='utf-8-sig') as output_file:
    dict_writer = csv.DictWriter(output_file, fieldnames=keys)
    dict_writer.writeheader()
    dict_writer.writerows(course_list)