model.py 3.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131
  1. import re
  2. import _html
  3. import _bs
  4. import pytz
  5. import datetime
  6. # import time
  7. import psycopg2.extras
  8. import view
  9. """
  10. Modules _html and _bs4 contain specialized methods.
  11. """
  12. local_timezones = {
  13. "NSW": "Australia/Sydney",
  14. "VIC": "Australia/Melbourne",
  15. "QLD": "Australia/Brisbane",
  16. "WA": "Australia/Perth",
  17. "SA": "Australia/Adelaide",
  18. "TAS": "Australia/Hobart",
  19. "ACT": "Australia/Sydney",
  20. "NT": "Australia/Darwin"}
  21. def scrape_main_page(row):
  22. this_url = """https://racingaustralia.horse/Home.aspx"""
  23. this_data = _html.get_page(this_url)
  24. venues_all = _bs.get_today_row(this_data, row)
  25. return venues_all
  26. def get_raw_scratchings(this_venue):
  27. this_raw_data = _html.get_page(this_venue.scratchings_url)
  28. return this_raw_data
  29. def process_raw_data(this_raw_data, this_venue):
  30. """
  31. Processes the raw data from the Scratchings page to obtain meta data.
  32. this_venue is passed to _bs.process_scratchings() to create the inherited namedTuple
  33. :param this_raw_data:
  34. :param this_venue:
  35. :return:
  36. """
  37. race_day_info = _bs.get_meta_data(this_raw_data, this_venue)
  38. return race_day_info
  39. def get_scratching_details(this_raw_data, this_venue):
  40. # this_data = _html.get_page(this_venue.scratchings_url)
  41. scratchings_info = _bs.process_scratchings(this_raw_data, this_venue)
  42. return scratchings_info
  43. def convert_to_unixtime(dt_object):
  44. """
  45. Simple utility function that returns the unixtime from a timezone aware dateTime object
  46. :param dt_object:
  47. :return:
  48. """
  49. utc = pytz.UTC
  50. d = dt_object.astimezone(utc)
  51. epoch = datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytz.UTC)
  52. ts = int((d - epoch).total_seconds())
  53. return ts
  54. def convert_to_date(weird_string):
  55. """
  56. Converts a string like 'MONDAY 15 JUL' to a python datetime object
  57. :param weird_string:
  58. :return datetime object:
  59. """
  60. weird_string = re.sub(r' (\d) ', ' 0\1 ', weird_string)
  61. local_timezone = pytz.timezone('Australia/Sydney')
  62. now = datetime.datetime.now(local_timezone)
  63. calculated_date = datetime.datetime.strptime(str(now.year) + ' ' + weird_string, "%Y %A %d %b").date()
  64. # print(calculated_date)
  65. return calculated_date
  66. def send_messages(messages):
  67. long_message = ''
  68. leftover_message = ''
  69. for this_message in messages:
  70. print('this_message: {}'.format(this_message))
  71. leftover_message = this_message
  72. # Append message if possible
  73. if len(long_message) + len(this_message) < 5997:
  74. if len(long_message) == 0:
  75. long_message = this_message
  76. else:
  77. long_message += '\n' + this_message
  78. leftover_message = ''
  79. else:
  80. # Send long message (max 6k characters)
  81. print('Sending very long message')
  82. view.broadcast(long_message)
  83. # Best would be to now store horses that were just broadcast
  84. long_message = this_message
  85. leftover_message = ''
  86. # Send all messages
  87. if len(long_message) > 0:
  88. print('Sending long_message')
  89. view.broadcast(long_message)
  90. long_message = ''
  91. # Send only or last message
  92. if len(leftover_message) > 0:
  93. print('Sending leftover_message')
  94. view.broadcast(leftover_message)
  95. leftover_message = ''
  96. def store_scratched_horses(horses, db):
  97. query = """INSERT INTO horses(venue, race_date, race, horse)
  98. VALUES(%s, %s, %s, %s)
  99. ON CONFLICT(venue, race_date, race, horse) DO NOTHING;"""
  100. cur3 = db.cursor(cursor_factory=psycopg2.extras.NamedTupleCursor)
  101. for database_entry in horses:
  102. cur3.execute(query, database_entry)
  103. print('Stored: {}'.format(database_entry))
  104. print(cur3.statusmessage)
  105. cur3.close()
  106. db.commit()