Script that I wrote when for scraping some agents info for a project from Life Happens.
Scraping was done for different zip codes.
Used C# .NET and Selenium on the project.
public class LifeAgentModel
{
[JsonIgnore]
public string name { get; set; }
public string zipCode { get; set; }
public string detail { get; set; }
public string company { get; set; }
public string phone { get; set; }
public string address { get; set; }
public string firstname { get; set; }
public string lastname { get; set; }
public string middlename { get; set; }
}
public static class WebDriverExtensions
{
public static IWebElement FindElement(this IWebDriver driver, By by, int timeoutInSeconds)
{
if (timeoutInSeconds > 0)
{
var wait = new WebDriverWait(driver, TimeSpan.FromSeconds(timeoutInSeconds));
return wait.Until(drv => drv.FindElement(by));
}
return driver.FindElement(by);
}
}
public class LifeHappensScraper
{
static void ScrapeAndAddItems(ChromeDriver driver, string zipCode)
{
var lsResult = new List<LifeAgentModel>();
driver.Navigate().GoToUrl("https://lifehappens.org/agent-locator/");
var input = driver.FindElement(By.XPath("//form[@class='zip']/input[@name='zip']"));
input.Click();
input.SendKeys(zipCode);
var input2 = driver.FindElement(By.XPath("//form[@class='zip']/div/input[@id='a']"));
input2.Click();
var input3 = driver.FindElement(By.XPath("//form[@class='zip']/input[@name='SubmitZip']"));
input3.Click();
driver.Manage().Timeouts().ImplicitWait = TimeSpan.FromSeconds(3);
var loadMoreLink = driver.FindElement(By.XPath("//a[@class='load-more']"));//
loadMoreLink.Click();
var listResult = driver.FindElement(By.XPath("//div[@id='result-1']"), 1);//
var agents = listResult.FindElements(By.XPath("//dt"));
var names = listResult.FindElements(By.XPath("//dt/div[@class='agent-info']/h3[@class='name']"));
var details = listResult.FindElements(By.XPath("//dt/div[@class='agent-info']/h3[@class='name']/span"));
var companies = listResult.FindElements(By.XPath("//dt/div[@class='agent-info']/span"));
var addresses = listResult.FindElements(By.XPath("//dt/a"));
var phones = listResult.FindElements(By.XPath("//dt/div[@class='tel']"));
//Click Load More button to handle client side pagination
var totalLoadMore = names.Count / 10;
if (names.Count % 10 > 0)
{
totalLoadMore = totalLoadMore + 1;
}
for (var i = 0; i < totalLoadMore; i++)
{
loadMoreLink = driver.FindElement(By.XPath("//a[@class='load-more']"));//
loadMoreLink.Click();
driver.Manage().Timeouts().ImplicitWait = TimeSpan.FromSeconds(2);
}
for (int i = 0; i < names.Count; i++)
{
var model = new LifeAgentModel()
{
name = names[i].Text,
detail = i >= details.Count ? "" : details[i].Text,
company = companies[i].Text,
address = addresses[i].Text.Length < 2 ? "..." : addresses[i].Text,
phone = phones[i].Text,
zipCode = zipCode
};
if (model.name.Length > 0)
{
var fullNames = model.name.Split(" ");
model.firstname = fullNames[0];
model.lastname = fullNames[1];
if (fullNames.Length == 3)
{
model.middlename = fullNames[1];
model.lastname = fullNames[2];
}
lsResult.Add(model);
}
}
}
