from selenium.common.exceptions import NoSuchElementException， WebDriverException

import numpy as np

import random

## helper function to consolidate two dictionaries

def merge_two_dicts(x， y)：

z = x.copy()   # start with x's keys and values

z.update(y)    # modifies z with y's keys and values & returns None

return z

scraped_data = []

## for each product in Tops category

for iterr in range(0，len(tops_link))：

init = 0

url = tops_link[iterr]

## open the URL in browser

try：

browser.get(url)

time.sleep(4)

except WebDriverException： ## when extracted URL is invalid

print('invalid url'， iterr)

continue

## get the webpage content

content = browser.page_source

soup = BeautifulSoup(content， "lxml")

## repeat until we run of review pages

while(True)：

## get the webpage content

content = browser.page_source

soup = BeautifulSoup(content， "lxml")

## extract reviewer details

reviewer_details = soup.find_all("div"， {"class"： "pr-rd-reviewer-details pr-rd-inner-side-content-block"})

## extract reviewers' name

reviewers_name = []

for reviewer in reviewer_details：

## In ModCloth， reviewer name appears as "By REVIEWER_NAME"

## Splitting at the end is to remove "By" to get only the actual reviewer name

reviewer_name = reviewer.find("p"， {"class"："pr-rd-details pr-rd-author-nickname"}).text.split('\n')[-1].strip()

reviewers_name.append(reviewer_name)

## extract "isVerified" information

isVerified = soup.find_all("span"， {"class"： "pr-rd-badging-text"})

## extract the fit feedback and customer measurements data (review_metadata)

review_data = soup.find_all("article"， {"class"： "pr-review"})

review_metadata_raw = []

for i in range(len(review_data))：

review_metadata_raw.append(review_data[i].find("div"， {"class"： "pr-accordion-content"}))

## extract HTML elements which contain review metadata

review_metadata_elements = [review_metadata_raw[i].find_all("dl"， {"class"， "pr-rd-def-list"})

if review_metadata_raw[i] is not None else None

for i in range(len(review_metadata_raw))]

## extract actual data from HTML elements

review_metadata = []

for element in review_metadata_elements：

if element is None：

review_metadata.append(None)

continue

## 
 elements contain metadata field name like "fit"， "length" etc

## 
 elements contain reviewer's response for those metadata fields like "small"， "just right" etc

review_metadata.append([(element[i].find("dt").text.lower()， element[i].find("dd").text.lower())

if element is not None else ""

for i in range(len(element))])

## extract review text

review_text = [txt.text for txt in soup.find_all("p"， {"class"： "pr-rd-description-text"})]

review_summary = [txt.text for txt in soup.find_all("h2"， {"class"： "pr-rd-review-headline"})]

## extract item id

item_id = soup.find("div"， {"class"： "product-number"}).find("span").text

## extract item category

try：

category = soup.find("a"， {"class"："breadcrumb-element"}).text.lower()

except AttributeError：  ## if category not present， item is not available

time.sleep(15 + random.randint(0，10))

break

## extract available product sizes

product_sizes = [i.text.strip().lower() for i in soup.find("ul"， {"class"： "swatches size"})

.find_all("li"， {"class"： "selectable variation-group-value"})]

item_info = {"category"： category， "item_id"： item_id， "product_sizes"： product_sizes}

## consolidate all the extracted data

## ignore records which don't have any review metadata as fit feedback is an essential signal for us

scraped_data.extend([merge_two_dicts({"review_text"： review_text[j]， "review_summary"： review_summary[j]}，

merge_two_dicts(merge_two_dicts({"user_name"：reviewers_name[j]}，

{data[0]：data[1] for data in review_metadata[j]})

，item_info))

for j in range(len(reviewer_details)) if review_metadata_raw[j] is not None])

## if current page is the initial one， it contains only NEXT button (PREVIOUS is missing)

if init == 0：

try：

init = 1

## execute click on NEXT by utilizing the xpath of NEXT

browser.execute_script("arguments[0].click();"，

browser.find_element_by_xpath('//*[@id="pr-review-display"]/footer/div/aside/button'))

time.sleep(10 + random.randint(0，5))

except NoSuchElementException： ## No NEXT button present， less than 10 reviews

time.sleep(15 + random.randint(0，10))

break

else：

try：

## execute click on NEXT by utilizing the xpath of NEXT

## if you notice， the xpath of NEXT is different here since PREVIOUS button is also present now

browser.execute_script("arguments[0].click();"，

browser.find_element_by_xpath('//*[@id="pr-review-display"]/footer/div/aside/button[2]'))

time.sleep(10 + random.randint(0，5))

except NoSuchElementException： ## No NEXT button， no more pages left

time.sleep(15 + random.randint(0，10))

break

## save the extracted data locally

np.save('./scraped_data_tops.npy'，scraped_data)

{

'bra size'： '42'，

'category'： 'tops'，

'cup size'： 'd'，

'fit'： 'slightly small'，

'height'： '5ft 6in'，

'hips'： '46.0'，

'item_id'： '149377'，

'length'： 'just right'，

'original_size'： 'xxxl'，

'product_sizes'： {'l'， 'm'， 's'， 'xl'， 'xs'， 'xxl'， 'xxxl'}，

'quality'： '4rated 4 out of 5 stars'，

'review_summary'： 'I love love love this shi'，

'review_text'： "I love love love this shirt. I ordered up because it looked a little more fitted in the picture and I'm glad I did; if I had ordered my normal size it would probably have been snugger than I prefer. The material is good qualityit's semithick. And the design is just so hilariously cute! I'm going to see if this brand has other tees and order more."，

'shoe size'： '8.00'，

'shoe width'： 'average'，

'size'： 16，

'user_name'： 'erinmbundren'

}