import re import _html import _bs import pytz import datetime # import time # import psycopg2.extras """ Modules _html and _bs4 contain specialized methods. """ local_timezones = { "NSW": "Australia/Sydney", "VIC": "Australia/Melbourne", "QLD": "Australia/Brisbane", "WA": "Australia/Perth", "SA": "Australia/Adelaide", "TAS": "Australia/Hobart", "ACT": "Australia/Sydney", "NT": "Australia/Darwin"} def scrape_main_page(row): this_url = """https://racingaustralia.horse/Home.aspx""" this_data = _html.get_page(this_url) venues_all = _bs.get_today_row(this_data, row) return venues_all def get_raw_scratchings(this_venue): this_raw_data = _html.get_page(this_venue.scratchings_url) return this_raw_data def process_raw_data(this_raw_data, this_venue): """ Processes the raw data from the Scratchings page to obtain meta data. this_venue is passed to _bs.process_scratchings() to create the inherited namedTuple :param this_raw_data: :param this_venue: :return: """ race_day_info = _bs.get_meta_data(this_raw_data, this_venue) return race_day_info def get_scratching_details(this_raw_data, this_venue): # this_data = _html.get_page(this_venue.scratchings_url) scratchings_info = _bs.process_scratchings(this_raw_data, this_venue) return scratchings_info def convert_to_unixtime(dt_object): """ Simple utility function that returns the unixtime from a timezone aware dateTime object :param dt_object: :return: """ utc = pytz.UTC d = dt_object.astimezone(utc) epoch = datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytz.UTC) ts = int((d - epoch).total_seconds()) return ts def convert_to_date(weird_string): """ Converts a string like 'MONDAY 15 JUL' to a python datetime object :param weird_string: :return datetime object: """ weird_string = re.sub(r' (\d) ', ' 0\1 ', weird_string) local_timezone = pytz.timezone('Australia/Sydney') now = datetime.datetime.now(local_timezone) calculated_date = datetime.datetime.strptime(str(now.year) + ' ' + weird_string, "%Y %A %d %b").date() # print(calculated_date) return calculated_date