import datetime import psycopg2.extras import model from pprint import pprint import time import database import sys import view if __name__ == '__main__': row = -1 if len(sys.argv) > 1: try: row = int(sys.argv[1]) except ValueError: sys.exit(1) broadcast = True if len(sys.argv) > 2: broadcast = False start = time.time() db = database.db race_days_global = model.scrape_main_page(row) interim = time.time() # print('interim 1 {}'.format(interim - start)) # pprint(race_days_global) race_days = [] raw_data_dict = {} for race_day in race_days_global: raw_data = model.get_raw_scratchings(race_day) race_day_details = model.process_raw_data(raw_data, race_day) race_days.append(race_day_details) raw_data_dict[race_day.name] = raw_data interim = time.time() # print('interim 2 {}'.format(interim - start)) # pprint(race_days) scratchings_for_database = [] messages = [] for race_day in race_days: if not raw_data_dict or not race_day: # raw_data_dict may be empty when there is no data available (yet) continue raw_data = raw_data_dict[race_day.name] scratchings = model.get_scratching_details(raw_data, race_day) if not scratchings: # model.get_scratchings_details may return empty continue # retrieve previous stored scratching for this venue / day query = "SELECT * FROM horses WHERE venue = %s AND race_date = %s;" db.commit() cur1 = db.cursor(cursor_factory=psycopg2.extras.NamedTupleCursor) cur1.execute(query, (race_day.name, race_day.date)) db_data_query1 = cur1.fetchall() # print(' 54. len(db_data): {}'.format(len(db_data))) # print(' 55. type(db_data): {}'.format(type(db_data))) # pprint(db_data) cur1.close() # compare retrieved scratchings with new data for scratching in scratchings: found_in_database = False # if db_data is None: # print(' 62. Caught db_data is None') # found_in_database = False # else: for row in db_data_query1: # print('row: ', end = '') # print(type(row)) # print(row) # if type(row) == 'datetime.time': # continue try: # if not hasattr(db_data, 'name'): # continue date_in_db_row_formatted = row.race_date.strftime('%Y-%m-%d') if ( scratching.date == date_in_db_row_formatted and scratching.venue == row.venue and int(scratching.race) == int(row.race) and scratching.horse == row.horse ): message_string = 'Horse found: date = {}, venue = {}, race = {}, horse = {}' message = message_string.format(scratching.date, scratching.venue, scratching.race, scratching.horse) # print(message) # print(row) found_in_database = True except AttributeError as ae: print(ae) print(' 94. row: ', end='') print(row) pprint(db_data_query1) continue if not found_in_database: # report new scratching date_object = datetime.datetime.strptime(scratching.date, "%Y-%m-%d").date() day_abbr = date_object.strftime('%a') query = """ SELECT start_time, utctime, torn FROM race_program WHERE race_date = %s AND venue = %s AND race = %s; """ cur2 = db.cursor(cursor_factory=psycopg2.extras.NamedTupleCursor) cur2.execute(query, (scratching.date, scratching.venue, scratching.race)) db_data_query2 = cur2.fetchone() cur2.close() if db_data_query2 is None or len(db_data_query2) == 0: print('106. cursor.execute(query, (scratching.date, scratching.venue, scratching.race))') print('107. cursor.execute({}, ({}, {}, {}))'.format( query, scratching.date, scratching.venue, scratching.race )) print('110. No race found') continue flag = '' if db_data_query2.torn: flag = 'FLAGGED!! ' message_string = '{}venue = {} {} {}-{} | race = {} starts at {} | {} UTC | horse = {}' message = message_string.format(flag, day_abbr, scratching.date, scratching.state, scratching.venue, scratching.race, db_data_query2.start_time.strftime('%H:%M'), db_data_query2.utctime.strftime('%H:%M'), scratching.horse) print(message) if broadcast or flag == 'FLAGGED!! ': messages.append(message) # view.broadcast(message) # store new scratching scratchings_for_database.append((scratching.venue, scratching.date, scratching.race, scratching.horse)) long_message = '' leftover_message = '' for this_message in messages: print('this_message: {}'.format(this_message)) leftover_message = this_message # Append message if possible if len(long_message) + len(this_message) < 5997: if len(long_message) == 0: long_message = this_message else: long_message += '\n' + this_message leftover_message = '' else: # Send long message (max 6k characters) print('Sending very long message') view.broadcast(long_message) # Best would be to now store horses that were just broadcast long_message = this_message leftover_message = '' # Send all messages if len(long_message) > 0: print('Sending long_message') view.broadcast(long_message) long_message = '' # Send only or last message if len(leftover_message) > 0: print('Sending leftover_message') view.broadcast(leftover_message) leftover_message = '' query = """INSERT INTO horses(venue, race_date, race, horse) VALUES(%s, %s, %s, %s) ON CONFLICT(venue, race_date, race, horse) DO NOTHING;""" cur3 = db.cursor(cursor_factory=psycopg2.extras.NamedTupleCursor) for database_entry in scratchings_for_database: cur3.execute(query, database_entry) print('Stored: {}'.format(database_entry)) print(cur3.statusmessage) cur3.close() db.commit() db.close() interim = time.time() # print('interim 3 {}'.format(interim - start))