import datetime import psycopg2.extras import model # from pprint import pprint import time import database import sys import view if __name__ == '__main__': row = -1 if len(sys.argv) > 1: try: row = int(sys.argv[1]) except ValueError: sys.exit(1) broadcast = True if len(sys.argv) > 2: broadcast = False start = time.time() db = database.db race_days_global = model.scrape_main_page(row) interim = time.time() # print('interim 1 {}'.format(interim - start)) # pprint(race_days_global) race_days = [] raw_data_dict = {} for race_day in race_days_global: raw_data = model.get_raw_scratchings(race_day) race_day_details = model.process_raw_data(raw_data, race_day) race_days.append(race_day_details) raw_data_dict[race_day.name] = raw_data interim = time.time() # print('interim 2 {}'.format(interim - start)) # pprint(race_days) cursor = db.cursor(cursor_factory=psycopg2.extras.NamedTupleCursor) for race_day in race_days: if not raw_data_dict or not race_day: # raw_data_dict may be empty when there is no data available (yet) continue raw_data = raw_data_dict[race_day.name] scratchings = model.get_scratching_details(raw_data, race_day) if not scratchings: # model.get_scratchings_details may return empty continue # retrieve previous stored scratching for this venue / day query = "SELECT * FROM horses WHERE venue = %s AND race_date = %s;" cursor.execute(query, (race_day.name, race_day.date)) db_data = cursor.fetchall() # compare retrieved scratchings with new data for scratching in scratchings: found_in_database = False for row in db_data: # print(row) if ( scratching.date == row.race_date.strftime('%Y-%m-%d') and scratching.venue == row.venue and int(scratching.race) == int(row.race) and scratching.horse == row.horse ): message = 'Horse found: date = {}, venue = {}, race = {}, horse = {}'.format(scratching.date, scratching.venue, scratching.race, scratching.horse) # print(message) # print(row) found_in_database = True if not found_in_database: # report new scratching date_object = datetime.datetime.strptime(scratching.date, "%Y-%m-%d").date() day_abbr = date_object.strftime('%a') query = """ SELECT start_time, utctime FROM race_program WHERE race_date = %s AND venue = %s AND race = %s; """ cursor.execute(query, (scratching.date, scratching.venue, scratching.race)) db_data = cursor.fetchone() message = 'venue = {} {} {}-{} | race = {} starts at {} | {} UTC | horse = {}'.format(day_abbr, scratching.date, scratching.state, scratching.venue, scratching.race, db_data.start_time, db_data.utctime, scratching.horse) print(message) if broadcast: view.broadcast(message) # store new scratching query = "INSERT INTO horses(venue, race_date, race, horse) VALUES(%s, %s, %s, %s)" cursor.execute(query, (scratching.venue, scratching.date, scratching.race, scratching.horse)) db.commit() time.sleep(0.5) cursor.close() db.close() interim = time.time() # print('interim 3 {}'.format(interim - start))