Qfyilyi

I want to retrieve the match data from the following website:

https://understat.com/match/81

I wrote the following script:

import sys

import time 

import os

import io

import csv



from selenium import webdriver

import selenium.webdriver.support.expected_conditions as ec

import selenium.webdriver.support.ui as ui

from selenium.common.exceptions import TimeoutException

from selenium.webdriver.common.by import By



driver = None

cnx = None

currentDir = sys.path[0]



def scrap_understat():

   init_browser('firefox')



   for i in range(80, 10080):

      try:

         driver.get('https://understat.com/match/' + str(i))

         time.sleep(1)

         if try_find_Element(driver, By.CLASS_NAME, 'error-code') is not None:

            continue



         data = get_match_data()

         save_data(data)



      except Exception as ex:

         log_this(ex)

         print(str(ex))



   close_browser()





def get_match_data():

   data = 



   teams = driver.find_elements(By.NAME, 'team')



   for team in teams:

       team.find_element(By.XPATH, 'following-sibling::*').click()

       time.sleep(1)

       players = driver.find_element(By.ID, 'match-rosters').find_element(By.TAG_NAME, 'tbody').find_elements(By.TAG_NAME, 'tr')

       for player in players:

           tds = player.find_elements(By.TAG_NAME, 'td')

           record = [tds[1].text, tds[2].text, tds[3].text, tds[4].text,

                  tds[5].text, tds[6].text, tds[7].text, tds[8].text, tds[9].text]

           data.append(record)



   return data





def save_data(data):

# save CSV

  csv_file_path = currentDir + '/output' + current_time + '.csv'

  file = None

  writer = None

  if not os.path.exists(csv_file_path):

     file = io.open(csv_file_path, 'w', newline='', encoding='ISO-8859-1')

     writer = csv.writer(file)

     writer.writerow(

        ['player', 'pos', 'min', 'sh', 'g', 'kp', 'a', 'xG',

         'xA'])

  else:

      file = io.open(csv_file_path, 'a', newline='', encoding='ISO-8859-1')

      writer = csv.writer(file)

  for record in data:

      writer.writerow(record)

      file.close()

The output of my script looks like this:

So, there is a problem with the xG- and xA-columns. I only want the lowerscript-part whereas the script takes all the text within the td How do I change my script to only include the first part? By inspecting the page elements, I see that the undesired part is called sub-class

Second question: How do I get the teamname declared as variable (Manchester United / Tottenham Hotspurs)

Try this one to avoid matching sub text:

record = [tds[1].text, tds[2].text, tds[3].text, tds[4].text,

                  tds[5].text, tds[6].text, tds[7].text,

          driver.execute_script('return arguments[0].firstChild.textContent', tds[8]), 

          driver.execute_script('return arguments[0].firstChild.textContent', tds[9])]

To get teamnames you can use

home = driver.find_element_by_xpath('//label[@for="team-home"]').text

away = driver.find_element_by_xpath('//label[@for="team-away"]').text

P.S. Consider to use Waits instead of time.sleep

It looks like you just need to remove the sup's:

driver.execute_script("$('sup').remove()")