import re import _html import _bs import pytz import datetime # import time import psycopg2.extras import view """ Modules _html and _bs4 contain specialized methods. """ local_timezones = { "NSW": "Australia/Sydney", "VIC": "Australia/Melbourne", "QLD": "Australia/Brisbane", "WA": "Australia/Perth", "SA": "Australia/Adelaide", "TAS": "Australia/Hobart", "ACT": "Australia/Sydney", "NT": "Australia/Darwin"} def scrape_main_page(row): this_url = """https://racingaustralia.horse/Home.aspx""" this_data = _html.get_page(this_url) venues_all = _bs.get_today_row(this_data, row) return venues_all def get_raw_scratchings(this_venue): this_raw_data = _html.get_page(this_venue.scratchings_url) return this_raw_data def process_raw_data(this_raw_data, this_venue): """ Processes the raw data from the Scratchings page to obtain meta data. this_venue is passed to _bs.process_scratchings() to create the inherited namedTuple :param this_raw_data: :param this_venue: :return: """ race_day_info = _bs.get_meta_data(this_raw_data, this_venue) return race_day_info def get_scratching_details(this_raw_data, this_venue): # this_data = _html.get_page(this_venue.scratchings_url) scratchings_info = _bs.process_scratchings(this_raw_data, this_venue) return scratchings_info def convert_to_unixtime(dt_object): """ Simple utility function that returns the unixtime from a timezone aware dateTime object :param dt_object: :return: """ utc = pytz.UTC d = dt_object.astimezone(utc) epoch = datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytz.UTC) ts = int((d - epoch).total_seconds()) return ts def convert_to_date(weird_string): """ Converts a string like 'MONDAY 15 JUL' to a python datetime object :param weird_string: :return datetime object: """ weird_string = re.sub(r' (\d) ', ' 0\1 ', weird_string) local_timezone = pytz.timezone('Australia/Sydney') now = datetime.datetime.now(local_timezone) calculated_date = datetime.datetime.strptime(str(now.year) + ' ' + weird_string, "%Y %A %d %b").date() # print(calculated_date) return calculated_date def send_messages(messages): long_message = '' leftover_message = '' for this_message in messages: print('this_message: {}'.format(this_message)) leftover_message = this_message # Append message if possible if len(long_message) + len(this_message) < 5997: if len(long_message) == 0: long_message = this_message else: long_message += '\n' + this_message leftover_message = '' else: # Send long message (max 6k characters) print('Sending very long message > {}'.format(len(long_message))) view.broadcast(long_message) # Best would be to now store horses that were just broadcast long_message = this_message leftover_message = '' # Send all messages if len(long_message) > 0: print('Sending long_message > {}'.format(len(long_message))) view.broadcast(long_message) long_message = '' # Send only or last message if len(leftover_message) > 0: print('Sending leftover_message > {}'.format(len(leftover_message))) view.broadcast(leftover_message) leftover_message = '' def store_scratched_horses(horses, db): query = """INSERT INTO horses(venue, race_date, race, horse) VALUES(%s, %s, %s, %s) ON CONFLICT(venue, race_date, race, horse) DO NOTHING;""" cur3 = db.cursor(cursor_factory=psycopg2.extras.NamedTupleCursor) for database_entry in horses: cur3.execute(query, database_entry) print('Stored: {}'.format(database_entry)) print(cur3.statusmessage) cur3.close() db.commit()