This is a snippet of Python’s Scrapy code that I recently wrote to crawl and scrape job listings from a Nepal’s leading job listing site MeroJob.
I am planning to use data scraped from this site to perform some data analysis on the current job market trend of Nepal.
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 |
def parseJobLink(self, response): item = MerojobItem() item['employer'] = response.xpath('//span[@itemprop="name"]/text()').extract_first() employerDesc = response.xpath('//div[@itemprop="description"]//text()').extract() item['employerDesc'] = response.xpath('//div[@itemprop="description"]//text()').extract_first().encode('ascii', 'ignore').decode('ascii').strip() employerDesc = response.xpath('//div[@itemprop="description"]//text()').extract() item['employerDesc'] = "".join(x.encode('ascii', 'ignore').decode('ascii').strip() for x in employerDesc) item['jobTitle'] = response.xpath('//h1[@itemprop="title"]/text()').extract_first() item['jobViews'] = response.xpath('//span[@class="text-success"]/text()').extract_first() item['jobViewsCountOn'] = str(date.today()) jobInfo = response.xpath('//table[@class="table table-hover table-no-border m-0"][1]') item['jobLink'] = response.url text = jobInfo.xpath('//tr/td[2]//text()').extract() itemText = response.xpath('//table[@class="table table-hover table-no-border m-0"][1]/tr/td[2]//text()').extract() if itemText is not None: str_list = [x.encode('ascii', 'ignore').decode('ascii').strip() for x in itemText if len(x.encode('ascii', 'ignore').decode('ascii').strip())>1] label = response.xpath('//table[@class="table table-hover table-no-border m-0"][1]/tr/td[1]//text()').extract() str_list_label = [x.encode('ascii', 'ignore').decode('ascii').strip() for x in label if len(x.encode('ascii', 'ignore').decode('ascii').strip())>1] numberOfVacancy = jobInfo.xpath('//tr[3]/td[2]/strong/text()').extract_first() item['numberOfVacancy'] = numberOfVacancy str_list_label = list(filter(lambda x : x != 'No. of Vacancy/s', str_list_label)) str_list_label = list(filter(lambda x : x != 'Apply Before', str_list_label)) for index, label in enumerate(str_list_label): if 'Category' in label: item['jobCategory'] = str_list[index] if 'Job Level' in label: item['jobLevel'] = str_list[index] if 'Employment Type' in label: item['employementType'] = str_list[index] if 'Salary' in label: item['salary'] = str_list[index] if 'Deadline' in label: item['applicationDeadline'] = str_list[index][:20].strip() if 'Education Level' in label: item['educationLevel'] = str_list[index] if 'Experience' in label: item['experienceRequired'] = str_list[index] yield item |
