Infinite scroll using selenium python

What can be the possible reasons of data not loading in an infinite scroll besides a slow internet connection using selenium python with javascript for infinite scroll?

Here is the code for getting the followers and followings list from koo app. It doesn't load data after the first 20 from the list of each.

import time
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
import csv
import json
import os
"""
from selenium.webdriver.chrome.options import Options
chrome_options = Options()
chrome_options.add_argument("--headless")
"""




def scroll(ele, driver, timeout):
    scroll_pause_time = timeout


    # Get scroll height
    last_height = driver.execute_script("return arguments[0].scrollHeight",ele)

    while True:
        # Scroll down to bottom
        driver.execute_script("arguments[0].scrollTo(0, arguments[0].scrollHeight, );",ele)
        
        # Wait to load page
        time.sleep(scroll_pause_time)       
        
        # Calculate new scroll height and compare with last scroll height
        new_height = driver.execute_script("return arguments[0].scrollHeight",ele)
        if new_height == last_height:
            # If heights are the same it will exit the function
            break
        last_height = new_height

            
# a list of user handles
users = []
curr_user =  0

with open("userList.json", "r") as read_it:
     users = json.load(read_it)

with open("curr_user.json", "r") as read_it:
     curr_user = json.load(read_it)

#print(users, curr_user)

"""
with open("userList.json", "w") as write_it:
     users = json.dump(users, write_it)
"""

       



#browser = webdriver.Chrome(options=chrome_options)
browser = webdriver.Chrome()

def count_network(name):

    browser.get("https://www.kooapp.com/profile/"+name)
    time.sleep(10)

    x1 = browser.find_element_by_class_name("Dbt8g")
    x2 = x1.find_element_by_tag_name("div")
    x3 = x2.find_elements_by_tag_name("div")
    x4 = x3[0].find_element_by_tag_name("a")
    x5 = x3[1].find_element_by_tag_name("a")

    followers_count = x4.find_element_by_tag_name("span").text
    following_count = x5.find_element_by_tag_name("span").text

    #print(name, followers_count, following_count)


    # using followers list
    followers_link = x4.get_attribute('href')
    followers_list = []

    browser.get(followers_link)
    time.sleep(10)

    
    ele = browser.find_element_by_id('scrollContainer')

    browser.execute_script("arguments[0].scrollIntoView();", ele)
    time.sleep(10)

    #load all the followers
    scroll(ele, browser, 20)

    #add all the user handles to the main list if not already present
    y1 = browser.find_elements_by_class_name("_2Bm2_")        
    for item in y1:
        y2 = item.find_elements_by_tag_name("div")[1].text
        y2 = y2[1:]
        followers_list.append(y2)

        #append it to the list only if it is not present
        if(users.count(y2) <= 0):
            users.append(y2)
            
    
    # returning back to the profile page
    browser.back()
    time.sleep(5)

    #get elements again since they get stale
    x1 = browser.find_element_by_class_name("Dbt8g")
    x2 = x1.find_element_by_tag_name("div")
    x3 = x2.find_elements_by_tag_name("div")
    x5 = x3[1].find_element_by_tag_name("a")

    # using following list
    following_link = x5.get_attribute('href')
    following_list = []

    browser.get(following_link)
    time.sleep(10)

    
    
    ele = browser.find_element_by_id('scrollContainer')

    browser.execute_script("arguments[0].scrollIntoView();", ele)
    time.sleep(10)

    #load all the followers
    scroll(ele, browser, 20)

    #add all the user handles to the main list if not already present
    y1 = browser.find_elements_by_class_name("_2Bm2_")
    for item in y1:
        y2 = item.find_elements_by_tag_name("div")[1].text
        y2 = y2[1:]
        following_list.append(y2)
        
        #append it to the list only if it is not present
        if(users.count(y2) <= 0):
            users.append(y2)

    with open('network_data.csv', 'a', encoding = "utf-8", newline = '') as file:
        wf = csv.writer(file)
        wf.writerow([name, followers_count, followers_list, following_count, following_list])
    


## start the loop for user handles
while(curr_user < len(users)):
    count_network(users[curr_user])
    curr_user += 1
    with open("userList.json", "w") as write_it:
        json.dump(users, write_it)

    with open("curr_user.json", "w") as write_it:
        json.dump(curr_user, write_it)


Read more here: https://stackoverflow.com/questions/68459041/infinite-scroll-using-selenium-python

Content Attribution

This content was originally published by coding zombie at Recent Questions - Stack Overflow, and is syndicated here via their RSS feed. You can read the original post over there.

%d bloggers like this: